From 43bb4e6b1f0227d48f11b03dcb8c497142745a32 Mon Sep 17 00:00:00 2001 From: Evgeniy Kochetov Date: Thu, 23 May 2019 12:04:49 +0300 Subject: [PATCH] rpc: Add NVMf transport statistics to nvmf_get_stats RPC method This patch adds transport part to nvmf_get_stats RPC method and basic infrastructure to report NVMf transport specific statistics. Signed-off-by: Evgeniy Kochetov Change-Id: Ie83b34f4ed932dd5f6d6e37897cf45228114bd88 Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/452299 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- CHANGELOG.md | 2 ++ doc/jsonrpc.md | 7 ++++- include/spdk/nvmf.h | 43 ++++++++++++++++++++++++++++ lib/event/subsystems/nvmf/nvmf_rpc.c | 34 ++++++++++++++++++++++ lib/nvmf/rdma.c | 36 +++++++++++++++++++++++ lib/nvmf/transport.c | 23 ++++++++++++++- lib/nvmf/transport.h | 15 ++++++++-- 7 files changed, 156 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c8b8a517..e3b36eadd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,8 @@ updated the related rpc function nvmf_create_transport to make this configurable parameter available to users. The `dif_insert_or_strip` is relevant for TCP transport for now and used to configure the DIF strip and insert. +Added infrastructure to retrieve NVMf transport statistics. + ### notify The function `spdk_notify_get_types()` and `spdk_notify_get_events()` were diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index b493dd16b..78964ee0c 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -4173,7 +4173,12 @@ Example response: "name": "app_thread", "admin_qpairs": 1, "io_qpairs": 4, - "pending_bdev_io": 1721 + "pending_bdev_io": 1721, + "transports": [ + { + "trtype": "RDMA" + } + ] } ] } diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index 39192744c..e1894e614 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -85,6 +85,15 @@ struct spdk_nvmf_poll_group_stat { uint64_t pending_bdev_io; }; +struct spdk_nvmf_transport_poll_group_stat { + spdk_nvme_transport_type_t trtype; + union { + struct { + int dummy; + } rdma; + }; +}; + /** * Construct an NVMe-oF target. * @@ -866,6 +875,40 @@ int spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport, void spdk_nvmf_tgt_transport_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt); + +/** + * \brief Get current transport poll group statistics. + * + * This function allocates memory for statistics and returns it + * in \p stat parameter. Caller must free this memory with + * spdk_nvmf_transport_poll_group_free_stat() when it is not needed + * anymore. + * + * \param tgt The NVMf target. + * \param transport The NVMf transport. + * \param stat Output parameter that will contain pointer to allocated statistics structure. + * + * \return 0 upon success. + * \return -ENOTSUP if transport does not support statistics. + * \return -EINVAL if any of parameters is NULL. + * \return -ENOENT if transport poll group is not found. + * \return -ENOMEM if memory allocation failed. + */ +int +spdk_nvmf_transport_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport *transport, + struct spdk_nvmf_transport_poll_group_stat **stat); + +/** + * Free statistics memory previously allocated with spdk_nvmf_transport_poll_group_get_stat(). + * + * \param transport The NVMf transport. + * \param stat Pointer to transport poll group statistics structure. + */ +void +spdk_nvmf_transport_poll_group_free_stat(struct spdk_nvmf_transport *transport, + struct spdk_nvmf_transport_poll_group_stat *stat); + #ifdef SPDK_CONFIG_RDMA /** * \brief Set the global hooks for the RDMA transport, if necessary. diff --git a/lib/event/subsystems/nvmf/nvmf_rpc.c b/lib/event/subsystems/nvmf/nvmf_rpc.c index 472c67839..bf31d8a01 100644 --- a/lib/event/subsystems/nvmf/nvmf_rpc.c +++ b/lib/event/subsystems/nvmf/nvmf_rpc.c @@ -1608,11 +1608,29 @@ rpc_nvmf_get_stats_done(struct spdk_io_channel_iter *i, int status) free(ctx); } +static void +write_nvmf_transport_stats(struct spdk_json_write_ctx *w, + struct spdk_nvmf_transport_poll_group_stat *stat) +{ + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "trtype", + spdk_nvme_transport_id_trtype_str(stat->trtype)); + switch (stat->trtype) { + case SPDK_NVME_TRANSPORT_RDMA: + default: + break; + } + spdk_json_write_object_end(w); +} + static void rpc_nvmf_get_stats(struct spdk_io_channel_iter *i) { struct rpc_nvmf_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + struct spdk_nvmf_transport *transport; struct spdk_nvmf_poll_group_stat stat; + struct spdk_nvmf_transport_poll_group_stat *trstat; + int rc; if (0 == spdk_nvmf_poll_group_get_stat(g_spdk_nvmf_tgt, &stat)) { spdk_json_write_object_begin(ctx->w); @@ -1620,6 +1638,22 @@ rpc_nvmf_get_stats(struct spdk_io_channel_iter *i) spdk_json_write_named_uint32(ctx->w, "admin_qpairs", stat.admin_qpairs); spdk_json_write_named_uint32(ctx->w, "io_qpairs", stat.io_qpairs); spdk_json_write_named_uint64(ctx->w, "pending_bdev_io", stat.pending_bdev_io); + + spdk_json_write_named_array_begin(ctx->w, "transports"); + transport = spdk_nvmf_transport_get_first(g_spdk_nvmf_tgt); + while (transport) { + rc = spdk_nvmf_transport_poll_group_get_stat(g_spdk_nvmf_tgt, transport, &trstat); + if (0 == rc) { + write_nvmf_transport_stats(ctx->w, trstat); + spdk_nvmf_transport_poll_group_free_stat(transport, trstat); + } else if (-ENOTSUP != rc) { + SPDK_ERRLOG("Failed to get poll group statistics for transport %s, errno %d\n", + spdk_nvme_transport_id_trtype_str(spdk_nvmf_get_transport_type(transport)), + rc); + } + transport = spdk_nvmf_transport_get_next(transport); + } + spdk_json_write_array_end(ctx->w); spdk_json_write_object_end(ctx->w); } diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 72c3bbf50..e87c764fa 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -3647,6 +3647,40 @@ spdk_nvmf_rdma_init_hooks(struct spdk_nvme_rdma_hooks *hooks) g_nvmf_hooks = *hooks; } +static int +spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport_poll_group_stat **stat) +{ + struct spdk_io_channel *ch; + struct spdk_nvmf_poll_group *group; + struct spdk_nvmf_transport_poll_group *tgroup; + + if (tgt == NULL || stat == NULL) { + return -EINVAL; + } + + ch = spdk_get_io_channel(tgt); + group = spdk_io_channel_get_ctx(ch);; + TAILQ_FOREACH(tgroup, &group->tgroups, link) { + if (SPDK_NVME_TRANSPORT_RDMA == tgroup->transport->ops->type) { + *stat = calloc(1, sizeof(struct spdk_nvmf_transport_poll_group_stat)); + if (!*stat) { + SPDK_ERRLOG("Failed to allocate memory for NVMf RDMA statistics\n"); + return -ENOMEM; + } + (*stat)->trtype = SPDK_NVME_TRANSPORT_RDMA; + return 0; + } + } + return -ENOENT; +} + +static void +spdk_nvmf_rdma_poll_group_free_stat(struct spdk_nvmf_transport_poll_group_stat *stat) +{ + free(stat); +} + const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma = { .type = SPDK_NVME_TRANSPORT_RDMA, .opts_init = spdk_nvmf_rdma_opts_init, @@ -3672,6 +3706,8 @@ const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma = { .qpair_get_local_trid = spdk_nvmf_rdma_qpair_get_local_trid, .qpair_get_listen_trid = spdk_nvmf_rdma_qpair_get_listen_trid, + .poll_group_get_stat = spdk_nvmf_rdma_poll_group_get_stat, + .poll_group_free_stat = spdk_nvmf_rdma_poll_group_free_stat, }; SPDK_LOG_REGISTER_COMPONENT("rdma", SPDK_LOG_RDMA) diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index 2b5ff4a1a..2490c7c0a 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) Intel Corporation. All rights reserved. - * Copyright (c) 2018 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2018-2019 Mellanox Technologies LTD. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -335,3 +335,24 @@ spdk_nvmf_transport_qpair_set_sqsize(struct spdk_nvmf_qpair *qpair) return 0; } + +int +spdk_nvmf_transport_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport *transport, + struct spdk_nvmf_transport_poll_group_stat **stat) +{ + if (transport->ops->poll_group_get_stat) { + return transport->ops->poll_group_get_stat(tgt, stat); + } else { + return -ENOTSUP; + } +} + +void +spdk_nvmf_transport_poll_group_free_stat(struct spdk_nvmf_transport *transport, + struct spdk_nvmf_transport_poll_group_stat *stat) +{ + if (transport->ops->poll_group_free_stat) { + transport->ops->poll_group_free_stat(stat); + } +} diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index fe0b76bc6..27b8693ad 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -1,8 +1,8 @@ /*- * BSD LICENSE * - * Copyright (c) Intel Corporation. - * All rights reserved. + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -167,6 +167,17 @@ struct spdk_nvmf_transport_ops { * set the submission queue size of the queue pair */ int (*qpair_set_sqsize)(struct spdk_nvmf_qpair *qpair); + + /* + * Get transport poll group statistics + */ + int (*poll_group_get_stat)(struct spdk_nvmf_tgt *tgt, + struct spdk_nvmf_transport_poll_group_stat **stat); + + /* + * Free transport poll group statistics previously allocated with poll_group_get_stat() + */ + void (*poll_group_free_stat)(struct spdk_nvmf_transport_poll_group_stat *stat); }; int spdk_nvmf_transport_stop_listen(struct spdk_nvmf_transport *transport,