bdev: a new public function to get the I/O statistics of bdev

Add a new function and its RPC caller. By using it, we can
get the statistics of all the bdevs or the specified bdev.

Meanwhile, with this patch, the open source tool 'sysstat/iostat'
can support for SPDK. The 'iostat' tool can call this function to
get the statistics of all the SPDK managed devices via the rpc
interface.

Change-Id: I135a7bbd49d923014bdf93720f78dd5a588d7afa
Signed-off-by: Yanbo Zhou <yanbo.zhou@intel.com>
Reviewed-on: https://review.gerrithub.io/393130
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

doc/jsonrpc.md

@@ -139,6 +139,54 @@ Example response:
 }
 ~~~
 
+## get_bdevs_iostat {#rpc_get_bdevs_iostat}
+
+Get I/O statistics of block devices (bdevs).
+
+### Parameters
+
+The user may specify no parameters in order to list all block devices, or a block device may be
+specified by name.
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+name                    | Optional | string      | Block device name
+
+### Response
+
+The response is an array of objects containing I/O statistics of the requested block devices.
+
+### Example
+
+Example request:
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "get_bdevs_iostat",
+  "params": {
+    "name": "Nvme0n1"
+  }
+}
+~~~
+
+Example response:
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": [
+    {
+      "name": "Nvme0n1",
+      "bytes_read": 34051522560,
+      "num_read_ops": 8312910,
+      "bytes_written": 0,
+      "num_write_ops": 0
+    }
+  ]
+}
+~~~
+
 ## delete_bdev {#rpc_delete_bdev}
 
 Unregister a block device.

include/spdk/bdev.h

@@ -125,6 +125,8 @@ struct spdk_bdev_io_stat {
 
 typedef void (*spdk_bdev_init_cb)(void *cb_arg, int rc);
 typedef void (*spdk_bdev_fini_cb)(void *cb_arg);
+typedef void (*spdk_bdev_get_device_stat_cb)(struct spdk_bdev *bdev,
+		struct spdk_bdev_io_stat *stat, void *cb_arg, int rc);
 
 /**
  * Initialize block device modules.
@@ -785,6 +787,17 @@ int spdk_bdev_free_io(struct spdk_bdev_io *bdev_io);
 void spdk_bdev_get_io_stat(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
 			   struct spdk_bdev_io_stat *stat);
 
+/**
+ * Return I/O statistics for this bdev. All the required information will be passed
+ * via the callback function.
+ *
+ * \param bdev Block device to query.
+ * \param cb Called when this operation completes.
+ * \param cb_arg Argument passed to callback function.
+ */
+void spdk_bdev_get_device_stat(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat,
+			       spdk_bdev_get_device_stat_cb cb, void *cb_arg);
+
 /**
  * Get the status of bdev_io as an NVMe status code.
  *

lib/bdev/bdev.c

@@ -217,6 +217,12 @@ struct spdk_bdev_desc {
 	TAILQ_ENTRY(spdk_bdev_desc)	link;
 };
 
+struct spdk_bdev_iostat_ctx {
+	struct spdk_bdev_io_stat *stat;
+	spdk_bdev_get_device_stat_cb cb;
+	void *cb_arg;
+};
+
 #define __bdev_to_io_dev(bdev)		(((char *)bdev) + 1)
 #define __bdev_from_io_dev(io_dev)	((struct spdk_bdev *)(((char *)io_dev) - 1))
 
@@ -1968,6 +1974,59 @@ spdk_bdev_get_io_stat(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
 	memset(&channel->stat, 0, sizeof(channel->stat));
 }
 
+static void
+_spdk_bdev_get_device_stat_done(struct spdk_io_channel_iter *i, int status)
+{
+	void *io_device = spdk_io_channel_iter_get_io_device(i);
+	struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i);
+
+	bdev_iostat_ctx->cb(__bdev_from_io_dev(io_device), bdev_iostat_ctx->stat,
+			    bdev_iostat_ctx->cb_arg, 0);
+	free(bdev_iostat_ctx);
+}
+
+static void
+_spdk_bdev_get_each_channel_stat(struct spdk_io_channel_iter *i)
+{
+	struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i);
+	struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
+	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
+
+	bdev_iostat_ctx->stat->bytes_read += channel->stat.bytes_read;
+	bdev_iostat_ctx->stat->num_read_ops += channel->stat.num_read_ops;
+	bdev_iostat_ctx->stat->bytes_written += channel->stat.bytes_written;
+	bdev_iostat_ctx->stat->num_write_ops += channel->stat.num_write_ops;
+
+	spdk_for_each_channel_continue(i, 0);
+}
+
+void
+spdk_bdev_get_device_stat(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat,
+			  spdk_bdev_get_device_stat_cb cb, void *cb_arg)
+{
+	struct spdk_bdev_iostat_ctx *bdev_iostat_ctx;
+
+	assert(bdev != NULL);
+	assert(stat != NULL);
+	assert(cb != NULL);
+
+	bdev_iostat_ctx = calloc(1, sizeof(struct spdk_bdev_iostat_ctx));
+	if (bdev_iostat_ctx == NULL) {
+		SPDK_ERRLOG("Unable to allocate memory for spdk_bdev_iostat_ctx\n");
+		cb(bdev, stat, cb_arg, -ENOMEM);
+		return;
+	}
+
+	bdev_iostat_ctx->stat = stat;
+	bdev_iostat_ctx->cb = cb;
+	bdev_iostat_ctx->cb_arg = cb_arg;
+
+	spdk_for_each_channel(__bdev_to_io_dev(bdev),
+			      _spdk_bdev_get_each_channel_stat,
+			      bdev_iostat_ctx,
+			      _spdk_bdev_get_device_stat_done);
+}
+
 int
 spdk_bdev_nvme_admin_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
 			      const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,

lib/bdev/rpc/bdev_rpc.c

@@ -38,6 +38,159 @@
 
 #include "spdk_internal/bdev.h"
 
+struct rpc_get_bdevs_iostat_ctx {
+	int bdev_count;
+	struct spdk_jsonrpc_request *request;
+	struct spdk_json_write_ctx *w;
+};
+
+static void
+spdk_rpc_get_bdevs_iostat_cb(struct spdk_bdev *bdev,
+			     struct spdk_bdev_io_stat *stat, void *cb_arg, int rc)
+{
+	struct rpc_get_bdevs_iostat_ctx *ctx = cb_arg;
+	struct spdk_json_write_ctx *w = ctx->w;
+	const char *bdev_name;
+
+	if (rc != 0) {
+		goto done;
+	}
+
+	bdev_name = spdk_bdev_get_name(bdev);
+	if (bdev_name != NULL) {
+		spdk_json_write_object_begin(w);
+
+		spdk_json_write_name(w, "name");
+		spdk_json_write_string(w, bdev_name);
+
+		spdk_json_write_name(w, "bytes_read");
+		spdk_json_write_uint64(w, stat->bytes_read);
+
+		spdk_json_write_name(w, "num_read_ops");
+		spdk_json_write_uint64(w, stat->num_read_ops);
+
+		spdk_json_write_name(w, "bytes_written");
+		spdk_json_write_uint64(w, stat->bytes_written);
+
+		spdk_json_write_name(w, "num_write_ops");
+		spdk_json_write_uint64(w, stat->num_write_ops);
+
+		spdk_json_write_object_end(w);
+	}
+
+done:
+	free(stat);
+	if (--ctx->bdev_count == 0) {
+		spdk_json_write_array_end(ctx->w);
+		spdk_jsonrpc_end_result(ctx->request, ctx->w);
+		free(ctx);
+	}
+}
+
+struct rpc_get_bdevs_iostat {
+	char *name;
+};
+
+static void
+free_rpc_get_bdevs_iostat(struct rpc_get_bdevs_iostat *r)
+{
+	free(r->name);
+}
+
+static const struct spdk_json_object_decoder rpc_get_bdevs_iostat_decoders[] = {
+	{"name", offsetof(struct rpc_get_bdevs_iostat, name), spdk_json_decode_string, true},
+};
+
+static void
+spdk_rpc_get_bdevs_iostat(struct spdk_jsonrpc_request *request,
+			  const struct spdk_json_val *params)
+{
+	struct rpc_get_bdevs_iostat req = {};
+	struct spdk_bdev *bdev = NULL;
+	struct spdk_json_write_ctx *w;
+	struct spdk_bdev_io_stat *stat;
+	struct rpc_get_bdevs_iostat_ctx *ctx;
+
+	if (params != NULL) {
+		if (spdk_json_decode_object(params, rpc_get_bdevs_iostat_decoders,
+					    SPDK_COUNTOF(rpc_get_bdevs_iostat_decoders),
+					    &req)) {
+			SPDK_ERRLOG("spdk_json_decode_object failed\n");
+			goto invalid;
+		} else {
+			if (req.name == NULL) {
+				SPDK_ERRLOG("missing name param\n");
+				goto invalid;
+			}
+
+			bdev = spdk_bdev_get_by_name(req.name);
+			if (bdev == NULL) {
+				SPDK_ERRLOG("bdev '%s' does not exist\n", req.name);
+				goto invalid;
+			}
+
+			free_rpc_get_bdevs_iostat(&req);
+		}
+	}
+
+	ctx = calloc(1, sizeof(struct rpc_get_bdevs_iostat_ctx));
+	if (ctx == NULL) {
+		SPDK_ERRLOG("Failed to allocate rpc_get_bdevs_iostat_ctx struct\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "No memory left");
+		return;
+	}
+
+	w = spdk_jsonrpc_begin_result(request);
+	if (w == NULL) {
+		free(ctx);
+		return;
+	}
+
+	/*
+	 * Increment initial bdev_count so that it will never reach 0 in the middle
+	 * of iterating.
+	 */
+	ctx->bdev_count++;
+	ctx->request = request;
+	ctx->w = w;
+
+	spdk_json_write_array_begin(w);
+
+	if (bdev != NULL) {
+		stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
+		if (stat == NULL) {
+			SPDK_ERRLOG("Failed to allocate rpc_get_bdevs_iostat_ctx struct\n");
+		} else {
+			ctx->bdev_count++;
+			spdk_bdev_get_device_stat(bdev, stat, spdk_rpc_get_bdevs_iostat_cb, ctx);
+		}
+	} else {
+		for (bdev = spdk_bdev_first(); bdev != NULL; bdev = spdk_bdev_next(bdev)) {
+			stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
+			if (stat == NULL) {
+				SPDK_ERRLOG("Failed to allocate spdk_bdev_io_stat struct\n");
+				break;
+			}
+			ctx->bdev_count++;
+			spdk_bdev_get_device_stat(bdev, stat, spdk_rpc_get_bdevs_iostat_cb, ctx);
+		}
+	}
+
+	if (--ctx->bdev_count == 0) {
+		spdk_json_write_array_end(w);
+		spdk_jsonrpc_end_result(request, w);
+		free(ctx);
+	}
+
+	return;
+
+invalid:
+	spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");
+
+	free_rpc_get_bdevs_iostat(&req);
+}
+SPDK_RPC_REGISTER("get_bdevs_iostat", spdk_rpc_get_bdevs_iostat, SPDK_RPC_RUNTIME)
+
 static void
 spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w,
 			struct spdk_bdev *bdev)

scripts/rpc.py

@@ -207,6 +207,15 @@ if __name__ == "__main__":
     p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False)
     p.set_defaults(func=get_bdevs_config)
 
+    @call_cmd
+    def get_bdevs_iostat(args):
+        print_dict(rpc.bdev.get_bdevs_iostat(args.client, args))
+
+    p = subparsers.add_parser(
+        'get_bdevs_iostat', help='Display current I/O statistics of all the blockdevs or required blockdev.')
+    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False)
+    p.set_defaults(func=get_bdevs_iostat)
+
     @call_cmd
     def delete_bdev(args):
         rpc.bdev.delete_bdev(args.client, args)

scripts/rpc/bdev.py

@@ -111,6 +111,13 @@ def get_bdevs_config(client, args):
     return client.call('get_bdevs_config', params)
 
 
+def get_bdevs_iostat(client, args):
+    params = {}
+    if args.name:
+        params['name'] = args.name
+    return client.call('get_bdevs_iostat', params)
+
+
 def delete_bdev(client, args):
     params = {'name': args.bdev_name}
     return client.call('delete_bdev', params)

test/unit/lib/bdev/bdev.c/bdev_ut.c

@@ -220,6 +220,35 @@ free_vbdev(struct spdk_bdev *bdev)
 	free(bdev);
 }
 
+static void
+get_device_stat_cb(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat, void *cb_arg, int rc)
+{
+	const char *bdev_name;
+
+	CU_ASSERT(bdev != NULL);
+	CU_ASSERT(rc == 0);
+	bdev_name = spdk_bdev_get_name(bdev);
+	CU_ASSERT_STRING_EQUAL(bdev_name, "bdev0");
+
+	free(stat);
+	free_bdev(bdev);
+}
+
+static void
+get_device_stat_test(void)
+{
+	struct spdk_bdev *bdev;
+	struct spdk_bdev_io_stat *stat;
+
+	bdev = allocate_bdev("bdev0");
+	stat = calloc(1, sizeof(struct spdk_bdev_io_stat));
+	if (stat == NULL) {
+		free_bdev(bdev);
+		return;
+	}
+	spdk_bdev_get_device_stat(bdev, stat, get_device_stat_cb, NULL);
+}
+
 static void
 open_write_test(void)
 {
@@ -543,7 +572,8 @@ main(int argc, char **argv)
 		CU_add_test(suite, "num_blocks_test", num_blocks_test) == NULL ||
 		CU_add_test(suite, "io_valid", io_valid_test) == NULL ||
 		CU_add_test(suite, "open_write", open_write_test) == NULL ||
-		CU_add_test(suite, "alias_add_del", alias_add_del_test) == NULL
+		CU_add_test(suite, "alias_add_del", alias_add_del_test) == NULL ||
+		CU_add_test(suite, "get_device_stat", get_device_stat_test) == NULL
 	) {
 		CU_cleanup_registry();
 		return CU_get_error();