nvme: Add functions to get/free poll group statistics

These are interface functions that can be used by
an application e.g. spdk_nvme_perf or bdev_nvme
library. The next patches will add usage of these
functions.

Change-Id: I33b88e0e713c2ea5967f9241885e3257c5070577
Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6300
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

CHANGELOG.md

@@ -69,6 +69,9 @@ independent SPDK processes are running on one node.  The filter function can
 then be implemented in these processes to decide which SSDs to probe based on
 the new SSD's PCI address.
 
+New functions `spdk_nvme_poll_group_get_stats` and `spdk_nvme_poll_group_free_stats`
+added. These functions allow to get transport statistics per NVME poll group.
+
 ### nvmf
 
 Removed the `spdk_nvmf_tgt_listen` and `spdk_nvmf_subsystem_add_ns` API.

include/spdk/nvme.h

@@ -472,6 +472,11 @@ struct spdk_nvme_transport_poll_group_stat {
 	};
 };
 
+struct spdk_nvme_poll_group_stat {
+	uint32_t num_transports;
+	struct spdk_nvme_transport_poll_group_stat **transport_stat;
+};
+
 /*
  * Controller support flags
  *
@@ -2337,6 +2342,28 @@ int64_t spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *gr
  */
 void *spdk_nvme_poll_group_get_ctx(struct spdk_nvme_poll_group *group);
 
+/**
+ * Retrieves transport statistics for the given poll group.
+ *
+ * Note: the structure returned by this function should later be freed with
+ * @b spdk_nvme_poll_group_free_stats function
+ *
+ * \param group Pointer to NVME poll group
+ * \param stats Double pointer to statistics to be filled by this function
+ * \return 0 on success or negated errno on failure
+ */
+int spdk_nvme_poll_group_get_stats(struct spdk_nvme_poll_group *group,
+				   struct spdk_nvme_poll_group_stat **stats);
+
+/**
+ * Frees poll group statistics retrieved using @b spdk_nvme_poll_group_get_stats function
+ *
+ * @param group Pointer to a poll group
+ * @param stat Pointer to statistics to be released
+ */
+void spdk_nvme_poll_group_free_stats(struct spdk_nvme_poll_group *group,
+				     struct spdk_nvme_poll_group_stat *stat);
+
 /**
  * Get the identify namespace data as defined by the NVMe specification.
  *

lib/nvme/nvme_internal.h

@@ -1339,6 +1339,8 @@ int nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *t
 					struct spdk_nvme_transport_poll_group_stat **stats);
 void nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
 		struct spdk_nvme_transport_poll_group_stat *stats);
+enum spdk_nvme_transport_type nvme_transport_get_trtype(const struct spdk_nvme_transport
+		*transport);
 /*
  * Below ref related functions must be called with the global
  *  driver lock held for the multi-process condition.

lib/nvme/nvme_poll_group.c

@@ -3,6 +3,7 @@
  *
  *   Copyright (c) Intel Corporation.
  *   All rights reserved.
+ *   Copyright (c) 2021 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -192,3 +193,81 @@ spdk_nvme_poll_group_destroy(struct spdk_nvme_poll_group *group)
 
 	return 0;
 }
+
+int
+spdk_nvme_poll_group_get_stats(struct spdk_nvme_poll_group *group,
+			       struct spdk_nvme_poll_group_stat **stats)
+{
+	struct spdk_nvme_transport_poll_group *tgroup;
+	struct spdk_nvme_poll_group_stat *result;
+	uint32_t transports_count = 0;
+	/* Not all transports used by this poll group may support statistics reporting */
+	uint32_t reported_stats_count = 0;
+	int rc;
+
+	assert(group);
+	assert(stats);
+
+	result = calloc(1, sizeof(*result));
+	if (!result) {
+		SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
+		return -ENOMEM;
+	}
+
+	STAILQ_FOREACH(tgroup, &group->tgroups, link) {
+		transports_count++;
+	}
+
+	result->transport_stat = calloc(transports_count, sizeof(*result->transport_stat));
+	if (!result->transport_stat) {
+		SPDK_ERRLOG("Failed to allocate memory for poll group statistics\n");
+		free(result);
+		return -ENOMEM;
+	}
+
+	STAILQ_FOREACH(tgroup, &group->tgroups, link) {
+		rc = nvme_transport_poll_group_get_stats(tgroup, &result->transport_stat[reported_stats_count]);
+		if (rc == 0) {
+			reported_stats_count++;
+		}
+	}
+
+	if (reported_stats_count == 0) {
+		free(result->transport_stat);
+		free(result);
+		SPDK_DEBUGLOG(nvme, "No transport statistics available\n");
+		return -ENOTSUP;
+	}
+
+	result->num_transports = reported_stats_count;
+	*stats = result;
+
+	return 0;
+}
+
+void
+spdk_nvme_poll_group_free_stats(struct spdk_nvme_poll_group *group,
+				struct spdk_nvme_poll_group_stat *stat)
+{
+	struct spdk_nvme_transport_poll_group *tgroup;
+	uint32_t i;
+	uint32_t freed_stats __attribute__((unused)) = 0;
+
+	assert(group);
+	assert(stat);
+
+	for (i = 0; i < stat->num_transports; i++) {
+		STAILQ_FOREACH(tgroup, &group->tgroups, link) {
+			if (nvme_transport_get_trtype(tgroup->transport) == stat->transport_stat[i]->trtype) {
+				nvme_transport_poll_group_free_stats(tgroup, stat->transport_stat[i]);
+				freed_stats++;
+				break;
+			}
+		}
+	}
+
+	assert(freed_stats == stat->num_transports);
+
+	free(stat->transport_stat);
+	free(stat);
+}

lib/nvme/nvme_transport.c

@@ -619,3 +619,8 @@ nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgro
 		tgroup->transport->ops.poll_group_free_stats(tgroup, stats);
 	}
 }
+
+enum spdk_nvme_transport_type nvme_transport_get_trtype(const struct spdk_nvme_transport *transport)
+{
+	return transport->ops.type;
+}

lib/nvme/spdk_nvme.map

@@ -168,6 +168,9 @@
 
 	spdk_nvme_map_prps;
 
+	spdk_nvme_poll_group_get_stats;
+	spdk_nvme_poll_group_free_stats;
+
 	# public functions from nvme_zns.h
 	spdk_nvme_zns_ns_get_data;
 	spdk_nvme_zns_ns_get_zone_size_sectors;

test/unit/lib/nvme/nvme_poll_group.c/nvme_poll_group_ut.c

@@ -3,6 +3,7 @@
  *
  *   Copyright (c) Intel Corporation.
  *   All rights reserved.
+ *   Copyright (c) 2021 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -36,6 +37,8 @@
 #include "nvme/nvme_poll_group.c"
 #include "common/lib/test_env.c"
 
+SPDK_LOG_REGISTER_COMPONENT(nvme)
+
 struct spdk_nvme_transport {
 	const char				name[32];
 	TAILQ_ENTRY(spdk_nvme_transport)	link;
@@ -68,6 +71,18 @@ DEFINE_STUB(nvme_transport_qpair_get_optimal_poll_group,
 	    (const struct spdk_nvme_transport *transport,
 	     struct spdk_nvme_qpair *qpair),
 	    NULL);
+DEFINE_STUB(nvme_transport_poll_group_get_stats,
+	    int,
+	    (struct spdk_nvme_transport_poll_group *tgroup,
+	     struct spdk_nvme_transport_poll_group_stat **stats),
+	    0);
+DEFINE_STUB_V(nvme_transport_poll_group_free_stats,
+	      (struct spdk_nvme_transport_poll_group *tgroup,
+	       struct spdk_nvme_transport_poll_group_stat *stats));
+DEFINE_STUB(nvme_transport_get_trtype,
+	    enum spdk_nvme_transport_type,
+	    (const struct spdk_nvme_transport *transport),
+	    SPDK_NVME_TRANSPORT_PCIE);
 
 static void
 unit_test_disconnected_qpair_cb(struct spdk_nvme_qpair *qpair, void *poll_group_ctx)