bdev: Add bdev_reset_iostat RPC

Add a helper function bdev_reset_device_stat() to reset I/O statistics.
This funciton is used for the bdev_reset_iostat RPC.

We do not have any plan to use bdev_reset_device_stat() outside
lib/bdev. Hence, we do not add this as a public API.

Then, add a new RPC bdev_reset_iostat to reset I/O statistics of a
single bdev or all bdevs.

Resetting I/O statistics affects all consumers. Add a note to CHANGELOG
and doc/jsonrpc.md.

Signed-off-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I97af09107b5c3ad1f9c19bf3cbf027457c4fbae7
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15350
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

CHANGELOG.md

@@ -40,6 +40,9 @@ Converted internal use of `pthread_mutex_t` to `struct spdk_spinlock`. Consumers
 API functions must be on an SPDK thread or the program will abort. It is now enforced
 that no internal bdev locks can be held when a poller or message goes off CPU.
 
+A new RPC `bdev_reset_iostat` was added to reset I/O statistics of bdevs. Note that if one
+consumer reset I/O statistics, it affects all other consumers.
+
 ### event
 
 Added core lock file mechanism to prevent the same CPU cores from being used by multiple

doc/jsonrpc.md

@@ -2119,6 +2119,45 @@ Example response:
 }
 ~~~
 
+### bdev_reset_iostat {#rpc_bdev_reset_iostat}
+
+Reset I/O statistics of block devices (bdevs). Note that if one consumer resets I/O statistics,
+it affects all other consumers.
+
+#### Parameters
+
+The user may specify no parameters in order to reset I/O statistics for all block devices, or
+a block device may be specified by name.
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+name                    | Optional | string      | Block device name
+
+#### Example
+
+Example request:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "bdev_reset_iostat",
+  "params": {
+    "name": "Nvme0n1"
+  }
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~
+
 ### bdev_enable_histogram {#rpc_bdev_enable_histogram}
 
 Control whether collecting data for histogram is enabled for specified bdev.

lib/bdev/bdev.c

@@ -3693,6 +3693,20 @@ bdev_io_stat_get(struct spdk_bdev_io_stat *to_stat, struct spdk_bdev_io_stat *fr
 	memcpy(to_stat, from_stat, sizeof(struct spdk_bdev_io_stat));
 }
 
+static void
+bdev_io_stat_reset(struct spdk_bdev_io_stat *stat)
+{
+	stat->bytes_read = 0;
+	stat->num_read_ops = 0;
+	stat->bytes_written = 0;
+	stat->num_write_ops = 0;
+	stat->bytes_unmapped = 0;
+	stat->num_unmap_ops = 0;
+	stat->read_latency_ticks = 0;
+	stat->write_latency_ticks = 0;
+	stat->unmap_latency_ticks = 0;
+}
+
 struct spdk_bdev_io_stat *
 bdev_io_stat_alloc(void)
 {
@@ -5688,6 +5702,60 @@ spdk_bdev_get_device_stat(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat
 				   bdev_get_device_stat_done);
 }
 
+struct bdev_iostat_reset_ctx {
+	bdev_reset_device_stat_cb cb;
+	void *cb_arg;
+};
+
+static void
+bdev_reset_device_stat_done(struct spdk_bdev *bdev, void *_ctx, int status)
+{
+	struct bdev_iostat_reset_ctx *ctx = _ctx;
+
+	ctx->cb(bdev, ctx->cb_arg, 0);
+
+	free(ctx);
+}
+
+static void
+bdev_reset_each_channel_stat(struct spdk_bdev_channel_iter *i, struct spdk_bdev *bdev,
+			     struct spdk_io_channel *ch, void *ctx)
+{
+	struct spdk_bdev_channel *channel = __io_ch_to_bdev_ch(ch);
+
+	bdev_io_stat_reset(channel->stat);
+
+	spdk_bdev_for_each_channel_continue(i, 0);
+}
+
+void
+bdev_reset_device_stat(struct spdk_bdev *bdev, bdev_reset_device_stat_cb cb, void *cb_arg)
+{
+	struct bdev_iostat_reset_ctx *ctx;
+
+	assert(bdev != NULL);
+	assert(cb != NULL);
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (ctx == NULL) {
+		SPDK_ERRLOG("Unable to allocate bdev_iostat_reset_ctx.\n");
+		cb(bdev, cb_arg, -ENOMEM);
+		return;
+	}
+
+	ctx->cb = cb;
+	ctx->cb_arg = cb_arg;
+
+	spdk_spin_lock(&bdev->internal.spinlock);
+	bdev_io_stat_reset(bdev->internal.stat);
+	spdk_spin_unlock(&bdev->internal.spinlock);
+
+	spdk_bdev_for_each_channel(bdev,
+				   bdev_reset_each_channel_stat,
+				   ctx,
+				   bdev_reset_device_stat_done);
+}
+
 int
 spdk_bdev_nvme_admin_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
 			      const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,

lib/bdev/bdev_internal.h

@@ -25,4 +25,9 @@ struct spdk_bdev_io_stat *bdev_io_stat_alloc(void);
 void bdev_io_stat_free(struct spdk_bdev_io_stat *stat);
 void bdev_io_stat_dump_json(struct spdk_bdev_io_stat *stat, struct spdk_json_write_ctx *w);
 
+typedef void (*bdev_reset_device_stat_cb)(struct spdk_bdev *bdev, void *cb_arg, int rc);
+
+void bdev_reset_device_stat(struct spdk_bdev *bdev, bdev_reset_device_stat_cb cb,
+			    void *cb_arg);
+
 #endif /* SPDK_BDEV_INTERNAL_H */

lib/bdev/bdev_rpc.c

@@ -457,6 +457,168 @@ rpc_bdev_get_iostat(struct spdk_jsonrpc_request *request,
 }
 SPDK_RPC_REGISTER("bdev_get_iostat", rpc_bdev_get_iostat, SPDK_RPC_RUNTIME)
 
+struct rpc_reset_iostat_ctx {
+	int bdev_count;
+	int rc;
+	struct spdk_jsonrpc_request *request;
+	struct spdk_json_write_ctx *w;
+};
+
+struct bdev_reset_iostat_ctx {
+	struct rpc_reset_iostat_ctx *rpc_ctx;
+	struct spdk_bdev_desc *desc;
+};
+
+static void
+rpc_reset_iostat_done(struct rpc_reset_iostat_ctx *rpc_ctx)
+{
+	if (--rpc_ctx->bdev_count != 0) {
+		return;
+	}
+
+	if (rpc_ctx->rc == 0) {
+		spdk_jsonrpc_send_bool_response(rpc_ctx->request, true);
+	} else {
+		spdk_jsonrpc_send_error_response(rpc_ctx->request, rpc_ctx->rc,
+						 spdk_strerror(-rpc_ctx->rc));
+	}
+
+	free(rpc_ctx);
+}
+
+static void
+bdev_reset_iostat_done(struct spdk_bdev *bdev, void *cb_arg, int rc)
+{
+	struct bdev_reset_iostat_ctx *bdev_ctx = cb_arg;
+	struct rpc_reset_iostat_ctx *rpc_ctx = bdev_ctx->rpc_ctx;
+
+	if (rc != 0 || rpc_ctx->rc != 0) {
+		if (rpc_ctx->rc == 0) {
+			rpc_ctx->rc = rc;
+		}
+	}
+
+	rpc_reset_iostat_done(rpc_ctx);
+
+	spdk_bdev_close(bdev_ctx->desc);
+	free(bdev_ctx);
+}
+
+static int
+bdev_reset_iostat(void *ctx, struct spdk_bdev *bdev)
+{
+	struct rpc_reset_iostat_ctx *rpc_ctx = ctx;
+	struct bdev_reset_iostat_ctx *bdev_ctx;
+	int rc;
+
+	bdev_ctx = calloc(1, sizeof(struct bdev_reset_iostat_ctx));
+	if (bdev_ctx == NULL) {
+		SPDK_ERRLOG("Failed to allocate bdev_iostat_ctx struct\n");
+		return -ENOMEM;
+	}
+
+	rc = spdk_bdev_open_ext(spdk_bdev_get_name(bdev), false, dummy_bdev_event_cb, NULL,
+				&bdev_ctx->desc);
+	if (rc != 0) {
+		free(bdev_ctx);
+		SPDK_ERRLOG("Failed to open bdev\n");
+		return rc;
+	}
+
+	rpc_ctx->bdev_count++;
+	bdev_ctx->rpc_ctx = rpc_ctx;
+	bdev_reset_device_stat(bdev, bdev_reset_iostat_done, bdev_ctx);
+
+	return 0;
+}
+
+struct rpc_bdev_reset_iostat {
+	char *name;
+};
+
+static void
+free_rpc_bdev_reset_iostat(struct rpc_bdev_reset_iostat *r)
+{
+	free(r->name);
+}
+
+static const struct spdk_json_object_decoder rpc_bdev_reset_iostat_decoders[] = {
+	{"name", offsetof(struct rpc_bdev_reset_iostat, name), spdk_json_decode_string, true},
+};
+
+static void
+rpc_bdev_reset_iostat(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params)
+{
+	struct rpc_bdev_reset_iostat req = {};
+	struct spdk_bdev_desc *desc = NULL;
+	struct rpc_reset_iostat_ctx *rpc_ctx;
+	struct bdev_reset_iostat_ctx *bdev_ctx;
+	int rc;
+
+	if (params != NULL) {
+		if (spdk_json_decode_object(params, rpc_bdev_reset_iostat_decoders,
+					    SPDK_COUNTOF(rpc_bdev_reset_iostat_decoders),
+					    &req)) {
+			SPDK_ERRLOG("spdk_json_decode_object failed\n");
+			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+							 "spdk_json_decode_object failed");
+			free_rpc_bdev_reset_iostat(&req);
+			return;
+		}
+
+		if (req.name) {
+			rc = spdk_bdev_open_ext(req.name, false, dummy_bdev_event_cb, NULL, &desc);
+			if (rc != 0) {
+				SPDK_ERRLOG("Failed to open bdev '%s': %d\n", req.name, rc);
+				spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
+				free_rpc_bdev_reset_iostat(&req);
+				return;
+			}
+		}
+	}
+
+	free_rpc_bdev_reset_iostat(&req);
+
+	rpc_ctx = calloc(1, sizeof(struct rpc_reset_iostat_ctx));
+	if (rpc_ctx == NULL) {
+		SPDK_ERRLOG("Failed to allocate rpc_iostat_ctx struct\n");
+		spdk_jsonrpc_send_error_response(request, -ENOMEM, spdk_strerror(ENOMEM));
+		return;
+	}
+
+	/*
+	 * Increment initial bdev_count so that it will never reach 0 in the middle
+	 * of iterating.
+	 */
+	rpc_ctx->bdev_count++;
+	rpc_ctx->request = request;
+
+	if (desc != NULL) {
+		bdev_ctx = calloc(1, sizeof(struct bdev_reset_iostat_ctx));
+		if (bdev_ctx == NULL) {
+			SPDK_ERRLOG("Failed to allocate bdev_iostat_ctx struct\n");
+			rpc_ctx->rc = -ENOMEM;
+
+			spdk_bdev_close(desc);
+		} else {
+			bdev_ctx->desc = desc;
+
+			rpc_ctx->bdev_count++;
+			bdev_ctx->rpc_ctx = rpc_ctx;
+			bdev_reset_device_stat(spdk_bdev_desc_get_bdev(desc),
+					       bdev_reset_iostat_done, bdev_ctx);
+		}
+	} else {
+		rc = spdk_for_each_bdev(rpc_ctx, bdev_reset_iostat);
+		if (rc != 0 && rpc_ctx->rc == 0) {
+			rpc_ctx->rc = rc;
+		}
+	}
+
+	rpc_reset_iostat_done(rpc_ctx);
+}
+SPDK_RPC_REGISTER("bdev_reset_iostat", rpc_bdev_reset_iostat, SPDK_RPC_RUNTIME)
+
 static int
 rpc_dump_bdev_info(void *ctx, struct spdk_bdev *bdev)
 {

python/spdk/rpc/bdev.py

@@ -1523,6 +1523,18 @@ def bdev_get_iostat(client, name=None, per_channel=None):
     return client.call('bdev_get_iostat', params)
 
 
+def bdev_reset_iostat(client, name=None):
+    """Reset I/O statistics for block devices.
+
+    Args:
+        name: bdev name to reset (optional; if omitted, reset all bdevs)
+    """
+    params = {}
+    if name:
+        params['name'] = name
+    return client.call('bdev_reset_iostat', params)
+
+
 def bdev_enable_histogram(client, name, enable):
     """Control whether histogram is enabled for specified bdev.
 

scripts/rpc.py

@@ -1135,12 +1135,20 @@ if __name__ == "__main__":
                                             per_channel=args.per_channel))
 
     p = subparsers.add_parser('bdev_get_iostat',
-                              help='Display current I/O statistics of all the blockdevs or required blockdev.')
+                              help='Display current I/O statistics of all the blockdevs or specified blockdev.')
     p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False)
     p.add_argument('-c', '--per-channel', default=False, dest='per_channel', help='Display per channel IO stats for specified device',
                    action='store_true', required=False)
     p.set_defaults(func=bdev_get_iostat)
 
+    def bdev_reset_iostat(args):
+        rpc.bdev.bdev_reset_iostat(args.client, name=args.name)
+
+    p = subparsers.add_parser('bdev_reset_iostat',
+                              help='Reset I/O statistics of all the blockdevs or specified blockdev.')
+    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False)
+    p.set_defaults(func=bdev_reset_iostat)
+
     def bdev_enable_histogram(args):
         rpc.bdev.bdev_enable_histogram(args.client, name=args.name, enable=args.enable)