From 64ccd4b95b154f7f8355c679c3521d564377fe2b Mon Sep 17 00:00:00 2001 From: Yanbo Zhou Date: Thu, 28 Dec 2017 17:03:17 +0800 Subject: [PATCH] bdev: a new public function to get the I/O statistics of bdev Add a new function and its RPC caller. By using it, we can get the statistics of all the bdevs or the specified bdev. Meanwhile, with this patch, the open source tool 'sysstat/iostat' can support for SPDK. The 'iostat' tool can call this function to get the statistics of all the SPDK managed devices via the rpc interface. Change-Id: I135a7bbd49d923014bdf93720f78dd5a588d7afa Signed-off-by: Yanbo Zhou Reviewed-on: https://review.gerrithub.io/393130 Tested-by: SPDK Automated Test System Reviewed-by: Dariusz Stojaczyk Reviewed-by: Daniel Verkamp Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker --- doc/jsonrpc.md | 48 +++++++++ include/spdk/bdev.h | 13 +++ lib/bdev/bdev.c | 59 +++++++++++ lib/bdev/rpc/bdev_rpc.c | 153 ++++++++++++++++++++++++++++ scripts/rpc.py | 9 ++ scripts/rpc/bdev.py | 7 ++ test/unit/lib/bdev/bdev.c/bdev_ut.c | 32 +++++- 7 files changed, 320 insertions(+), 1 deletion(-) diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index cc85d6166..b1250d1f4 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -139,6 +139,54 @@ Example response: } ~~~ +## get_bdevs_iostat {#rpc_get_bdevs_iostat} + +Get I/O statistics of block devices (bdevs). + +### Parameters + +The user may specify no parameters in order to list all block devices, or a block device may be +specified by name. + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Optional | string | Block device name + +### Response + +The response is an array of objects containing I/O statistics of the requested block devices. + +### Example + +Example request: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "get_bdevs_iostat", + "params": { + "name": "Nvme0n1" + } +} +~~~ + +Example response: +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "name": "Nvme0n1", + "bytes_read": 34051522560, + "num_read_ops": 8312910, + "bytes_written": 0, + "num_write_ops": 0 + } + ] +} +~~~ + ## delete_bdev {#rpc_delete_bdev} Unregister a block device. diff --git a/include/spdk/bdev.h b/include/spdk/bdev.h index 18f58d2d2..b6d77fc32 100644 --- a/include/spdk/bdev.h +++ b/include/spdk/bdev.h @@ -125,6 +125,8 @@ struct spdk_bdev_io_stat { typedef void (*spdk_bdev_init_cb)(void *cb_arg, int rc); typedef void (*spdk_bdev_fini_cb)(void *cb_arg); +typedef void (*spdk_bdev_get_device_stat_cb)(struct spdk_bdev *bdev, + struct spdk_bdev_io_stat *stat, void *cb_arg, int rc); /** * Initialize block device modules. @@ -785,6 +787,17 @@ int spdk_bdev_free_io(struct spdk_bdev_io *bdev_io); void spdk_bdev_get_io_stat(struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io_stat *stat); +/** + * Return I/O statistics for this bdev. All the required information will be passed + * via the callback function. + * + * \param bdev Block device to query. + * \param cb Called when this operation completes. + * \param cb_arg Argument passed to callback function. + */ +void spdk_bdev_get_device_stat(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat, + spdk_bdev_get_device_stat_cb cb, void *cb_arg); + /** * Get the status of bdev_io as an NVMe status code. * diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index c8b973883..006d46cea 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -217,6 +217,12 @@ struct spdk_bdev_desc { TAILQ_ENTRY(spdk_bdev_desc) link; }; +struct spdk_bdev_iostat_ctx { + struct spdk_bdev_io_stat *stat; + spdk_bdev_get_device_stat_cb cb; + void *cb_arg; +}; + #define __bdev_to_io_dev(bdev) (((char *)bdev) + 1) #define __bdev_from_io_dev(io_dev) ((struct spdk_bdev *)(((char *)io_dev) - 1)) @@ -1968,6 +1974,59 @@ spdk_bdev_get_io_stat(struct spdk_bdev *bdev, struct spdk_io_channel *ch, memset(&channel->stat, 0, sizeof(channel->stat)); } +static void +_spdk_bdev_get_device_stat_done(struct spdk_io_channel_iter *i, int status) +{ + void *io_device = spdk_io_channel_iter_get_io_device(i); + struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i); + + bdev_iostat_ctx->cb(__bdev_from_io_dev(io_device), bdev_iostat_ctx->stat, + bdev_iostat_ctx->cb_arg, 0); + free(bdev_iostat_ctx); +} + +static void +_spdk_bdev_get_each_channel_stat(struct spdk_io_channel_iter *i) +{ + struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i); + struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); + struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch); + + bdev_iostat_ctx->stat->bytes_read += channel->stat.bytes_read; + bdev_iostat_ctx->stat->num_read_ops += channel->stat.num_read_ops; + bdev_iostat_ctx->stat->bytes_written += channel->stat.bytes_written; + bdev_iostat_ctx->stat->num_write_ops += channel->stat.num_write_ops; + + spdk_for_each_channel_continue(i, 0); +} + +void +spdk_bdev_get_device_stat(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat, + spdk_bdev_get_device_stat_cb cb, void *cb_arg) +{ + struct spdk_bdev_iostat_ctx *bdev_iostat_ctx; + + assert(bdev != NULL); + assert(stat != NULL); + assert(cb != NULL); + + bdev_iostat_ctx = calloc(1, sizeof(struct spdk_bdev_iostat_ctx)); + if (bdev_iostat_ctx == NULL) { + SPDK_ERRLOG("Unable to allocate memory for spdk_bdev_iostat_ctx\n"); + cb(bdev, stat, cb_arg, -ENOMEM); + return; + } + + bdev_iostat_ctx->stat = stat; + bdev_iostat_ctx->cb = cb; + bdev_iostat_ctx->cb_arg = cb_arg; + + spdk_for_each_channel(__bdev_to_io_dev(bdev), + _spdk_bdev_get_each_channel_stat, + bdev_iostat_ctx, + _spdk_bdev_get_device_stat_done); +} + int spdk_bdev_nvme_admin_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes, diff --git a/lib/bdev/rpc/bdev_rpc.c b/lib/bdev/rpc/bdev_rpc.c index bc115d1a9..9589c104c 100644 --- a/lib/bdev/rpc/bdev_rpc.c +++ b/lib/bdev/rpc/bdev_rpc.c @@ -38,6 +38,159 @@ #include "spdk_internal/bdev.h" +struct rpc_get_bdevs_iostat_ctx { + int bdev_count; + struct spdk_jsonrpc_request *request; + struct spdk_json_write_ctx *w; +}; + +static void +spdk_rpc_get_bdevs_iostat_cb(struct spdk_bdev *bdev, + struct spdk_bdev_io_stat *stat, void *cb_arg, int rc) +{ + struct rpc_get_bdevs_iostat_ctx *ctx = cb_arg; + struct spdk_json_write_ctx *w = ctx->w; + const char *bdev_name; + + if (rc != 0) { + goto done; + } + + bdev_name = spdk_bdev_get_name(bdev); + if (bdev_name != NULL) { + spdk_json_write_object_begin(w); + + spdk_json_write_name(w, "name"); + spdk_json_write_string(w, bdev_name); + + spdk_json_write_name(w, "bytes_read"); + spdk_json_write_uint64(w, stat->bytes_read); + + spdk_json_write_name(w, "num_read_ops"); + spdk_json_write_uint64(w, stat->num_read_ops); + + spdk_json_write_name(w, "bytes_written"); + spdk_json_write_uint64(w, stat->bytes_written); + + spdk_json_write_name(w, "num_write_ops"); + spdk_json_write_uint64(w, stat->num_write_ops); + + spdk_json_write_object_end(w); + } + +done: + free(stat); + if (--ctx->bdev_count == 0) { + spdk_json_write_array_end(ctx->w); + spdk_jsonrpc_end_result(ctx->request, ctx->w); + free(ctx); + } +} + +struct rpc_get_bdevs_iostat { + char *name; +}; + +static void +free_rpc_get_bdevs_iostat(struct rpc_get_bdevs_iostat *r) +{ + free(r->name); +} + +static const struct spdk_json_object_decoder rpc_get_bdevs_iostat_decoders[] = { + {"name", offsetof(struct rpc_get_bdevs_iostat, name), spdk_json_decode_string, true}, +}; + +static void +spdk_rpc_get_bdevs_iostat(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_get_bdevs_iostat req = {}; + struct spdk_bdev *bdev = NULL; + struct spdk_json_write_ctx *w; + struct spdk_bdev_io_stat *stat; + struct rpc_get_bdevs_iostat_ctx *ctx; + + if (params != NULL) { + if (spdk_json_decode_object(params, rpc_get_bdevs_iostat_decoders, + SPDK_COUNTOF(rpc_get_bdevs_iostat_decoders), + &req)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + goto invalid; + } else { + if (req.name == NULL) { + SPDK_ERRLOG("missing name param\n"); + goto invalid; + } + + bdev = spdk_bdev_get_by_name(req.name); + if (bdev == NULL) { + SPDK_ERRLOG("bdev '%s' does not exist\n", req.name); + goto invalid; + } + + free_rpc_get_bdevs_iostat(&req); + } + } + + ctx = calloc(1, sizeof(struct rpc_get_bdevs_iostat_ctx)); + if (ctx == NULL) { + SPDK_ERRLOG("Failed to allocate rpc_get_bdevs_iostat_ctx struct\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "No memory left"); + return; + } + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + free(ctx); + return; + } + + /* + * Increment initial bdev_count so that it will never reach 0 in the middle + * of iterating. + */ + ctx->bdev_count++; + ctx->request = request; + ctx->w = w; + + spdk_json_write_array_begin(w); + + if (bdev != NULL) { + stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); + if (stat == NULL) { + SPDK_ERRLOG("Failed to allocate rpc_get_bdevs_iostat_ctx struct\n"); + } else { + ctx->bdev_count++; + spdk_bdev_get_device_stat(bdev, stat, spdk_rpc_get_bdevs_iostat_cb, ctx); + } + } else { + for (bdev = spdk_bdev_first(); bdev != NULL; bdev = spdk_bdev_next(bdev)) { + stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); + if (stat == NULL) { + SPDK_ERRLOG("Failed to allocate spdk_bdev_io_stat struct\n"); + break; + } + ctx->bdev_count++; + spdk_bdev_get_device_stat(bdev, stat, spdk_rpc_get_bdevs_iostat_cb, ctx); + } + } + + if (--ctx->bdev_count == 0) { + spdk_json_write_array_end(w); + spdk_jsonrpc_end_result(request, w); + free(ctx); + } + + return; + +invalid: + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters"); + + free_rpc_get_bdevs_iostat(&req); +} +SPDK_RPC_REGISTER("get_bdevs_iostat", spdk_rpc_get_bdevs_iostat, SPDK_RPC_RUNTIME) + static void spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w, struct spdk_bdev *bdev) diff --git a/scripts/rpc.py b/scripts/rpc.py index 2750a54c4..41178e7a1 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -207,6 +207,15 @@ if __name__ == "__main__": p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False) p.set_defaults(func=get_bdevs_config) + @call_cmd + def get_bdevs_iostat(args): + print_dict(rpc.bdev.get_bdevs_iostat(args.client, args)) + + p = subparsers.add_parser( + 'get_bdevs_iostat', help='Display current I/O statistics of all the blockdevs or required blockdev.') + p.add_argument('-b', '--name', help="Name of the Blockdev. Example: Nvme0n1", required=False) + p.set_defaults(func=get_bdevs_iostat) + @call_cmd def delete_bdev(args): rpc.bdev.delete_bdev(args.client, args) diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 345fc1193..ecf372c86 100755 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -111,6 +111,13 @@ def get_bdevs_config(client, args): return client.call('get_bdevs_config', params) +def get_bdevs_iostat(client, args): + params = {} + if args.name: + params['name'] = args.name + return client.call('get_bdevs_iostat', params) + + def delete_bdev(client, args): params = {'name': args.bdev_name} return client.call('delete_bdev', params) diff --git a/test/unit/lib/bdev/bdev.c/bdev_ut.c b/test/unit/lib/bdev/bdev.c/bdev_ut.c index 51e96955d..351a0677b 100644 --- a/test/unit/lib/bdev/bdev.c/bdev_ut.c +++ b/test/unit/lib/bdev/bdev.c/bdev_ut.c @@ -220,6 +220,35 @@ free_vbdev(struct spdk_bdev *bdev) free(bdev); } +static void +get_device_stat_cb(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat, void *cb_arg, int rc) +{ + const char *bdev_name; + + CU_ASSERT(bdev != NULL); + CU_ASSERT(rc == 0); + bdev_name = spdk_bdev_get_name(bdev); + CU_ASSERT_STRING_EQUAL(bdev_name, "bdev0"); + + free(stat); + free_bdev(bdev); +} + +static void +get_device_stat_test(void) +{ + struct spdk_bdev *bdev; + struct spdk_bdev_io_stat *stat; + + bdev = allocate_bdev("bdev0"); + stat = calloc(1, sizeof(struct spdk_bdev_io_stat)); + if (stat == NULL) { + free_bdev(bdev); + return; + } + spdk_bdev_get_device_stat(bdev, stat, get_device_stat_cb, NULL); +} + static void open_write_test(void) { @@ -543,7 +572,8 @@ main(int argc, char **argv) CU_add_test(suite, "num_blocks_test", num_blocks_test) == NULL || CU_add_test(suite, "io_valid", io_valid_test) == NULL || CU_add_test(suite, "open_write", open_write_test) == NULL || - CU_add_test(suite, "alias_add_del", alias_add_del_test) == NULL + CU_add_test(suite, "alias_add_del", alias_add_del_test) == NULL || + CU_add_test(suite, "get_device_stat", get_device_stat_test) == NULL ) { CU_cleanup_registry(); return CU_get_error();