diff --git a/CHANGELOG.md b/CHANGELOG.md index 52758736f..841ae7065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,8 @@ were added to process I/O statistics outside the generic bdev layer, especially Added I/O statistics per I/O path to the NVMe bdev module for NVMe bdev multipath. It can be enabled by a new option io_path_stat of RPC bdev_nvme_set_options. +Added RPC bdev_nvme_get_path_iostat to get I/O statistics for IO paths of the NVMe bdev. + ### event Added core lock file mechanism to prevent the same CPU cores from being used by multiple diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 9fe8948cd..4df54849b 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -4211,6 +4211,102 @@ Example response: } ~~~ +### bdev_nvme_get_path_iostat {#rpc_bdev_nvme_get_path_iostat} + +Get I/O statistics for IO paths of the block device. Call RPC bdev_nvme_set_options to set enable_io_path_stat +true before using this RPC. + +#### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +name | Required | string | Name of the NVMe bdev + +#### Example + +Example request: + +~~~json +{ + "jsonrpc": "2.0", + "method": "bdev_nvme_get_path_iostat", + "id": 1, + "params": { + "name": "NVMe0n1" + } +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "name": "NVMe0n1", + "stats": [ + { + "trid": { + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "10.169.204.201", + "trsvcid": "4420", + "subnqn": "nqn.2016-06.io.spdk:cnode1" + }, + "stat": { + "bytes_read": 676691968, + "num_read_ops": 165201, + "bytes_written": 0, + "num_write_ops": 0, + "bytes_unmapped": 0, + "num_unmap_ops": 0, + "max_read_latency_ticks": 521487, + "min_read_latency_ticks": 0, + "write_latency_ticks": 0, + "max_write_latency_ticks": 0, + "min_write_latency_ticks": 0, + "unmap_latency_ticks": 0, + "max_unmap_latency_ticks": 0, + "min_unmap_latency_ticks": 0, + "copy_latency_ticks": 0, + "max_copy_latency_ticks": 0, + "min_copy_latency_ticks": 0 + } + }, + { + "trid": { + "trtype": "TCP", + "adrfam": "IPv4", + "traddr": "8.8.8.6", + "trsvcid": "4420", + "subnqn": "nqn.2016-06.io.spdk:cnode1" + }, + "stat": { + "bytes_read": 677138432, + "num_read_ops": 165317, + "bytes_written": 0, + "num_write_ops": 0, + "bytes_unmapped": 0, + "num_unmap_ops": 0, + "max_read_latency_ticks": 108525, + "min_read_latency_ticks": 0, + "write_latency_ticks": 0, + "max_write_latency_ticks": 0, + "min_write_latency_ticks": 0, + "unmap_latency_ticks": 0, + "max_unmap_latency_ticks": 0, + "min_unmap_latency_ticks": 0, + "copy_latency_ticks": 0, + "max_copy_latency_ticks": 0, + "min_copy_latency_ticks": 0 + } + } + ] + } +} +~~~ + ### bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register} Register CUSE device on NVMe controller. diff --git a/module/bdev/nvme/bdev_nvme_rpc.c b/module/bdev/nvme/bdev_nvme_rpc.c index a2f3cc2fe..1d32c3db5 100644 --- a/module/bdev/nvme/bdev_nvme_rpc.c +++ b/module/bdev/nvme/bdev_nvme_rpc.c @@ -2456,3 +2456,201 @@ rpc_bdev_nvme_get_mdns_discovery_info(struct spdk_jsonrpc_request *request, SPDK_RPC_REGISTER("bdev_nvme_get_mdns_discovery_info", rpc_bdev_nvme_get_mdns_discovery_info, SPDK_RPC_RUNTIME) + +struct rpc_get_path_stat { + char *name; +}; + +struct path_stat { + struct spdk_bdev_io_stat stat; + struct spdk_nvme_transport_id trid; + struct nvme_ns *ns; +}; + +struct rpc_bdev_nvme_path_stat_ctx { + struct spdk_jsonrpc_request *request; + struct path_stat *path_stat; + uint32_t num_paths; + struct spdk_bdev_desc *desc; +}; + +static void +free_rpc_get_path_stat(struct rpc_get_path_stat *req) +{ + free(req->name); +} + +static const struct spdk_json_object_decoder rpc_get_path_stat_decoders[] = { + {"name", offsetof(struct rpc_get_path_stat, name), spdk_json_decode_string}, +}; + +static void +dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) +{ +} + +static void +rpc_bdev_nvme_path_stat_per_channel(struct spdk_io_channel_iter *i) +{ + struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); + struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch); + struct nvme_io_path *io_path; + struct path_stat *path_stat; + uint32_t j; + + assert(ctx->num_paths != 0); + + for (j = 0; j < ctx->num_paths; j++) { + path_stat = &ctx->path_stat[j]; + + STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) { + if (path_stat->ns == io_path->nvme_ns) { + assert(io_path->stat != NULL); + spdk_bdev_add_io_stat(&path_stat->stat, io_path->stat); + } + } + } + + spdk_for_each_channel_continue(i, 0); +} + +static void +rpc_bdev_nvme_path_stat_done(struct spdk_io_channel_iter *i, int status) +{ + struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i); + struct spdk_json_write_ctx *w; + struct path_stat *path_stat; + uint32_t j; + + assert(ctx->num_paths != 0); + + w = spdk_jsonrpc_begin_result(ctx->request); + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "name", nbdev->disk.name); + spdk_json_write_named_array_begin(w, "stats"); + + for (j = 0; j < ctx->num_paths; j++) { + path_stat = &ctx->path_stat[j]; + spdk_json_write_object_begin(w); + + spdk_json_write_named_object_begin(w, "trid"); + nvme_bdev_dump_trid_json(&path_stat->trid, w); + spdk_json_write_object_end(w); + + spdk_json_write_named_object_begin(w, "stat"); + spdk_bdev_dump_io_stat_json(&path_stat->stat, w); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + } + + spdk_json_write_array_end(w); + spdk_json_write_object_end(w); + spdk_jsonrpc_end_result(ctx->request, w); + + spdk_bdev_close(ctx->desc); + free(ctx->path_stat); + free(ctx); +} + +static void +rpc_bdev_nvme_get_path_iostat(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_get_path_stat req = {}; + struct spdk_bdev_desc *desc = NULL; + struct spdk_bdev *bdev; + struct nvme_bdev *nbdev; + struct nvme_ns *nvme_ns; + struct path_stat *path_stat; + struct rpc_bdev_nvme_path_stat_ctx *ctx; + struct spdk_bdev_nvme_opts opts; + uint32_t num_paths = 0, i = 0; + int rc; + + bdev_nvme_get_opts(&opts); + if (!opts.io_path_stat) { + SPDK_ERRLOG("RPC not enabled if enable_io_path_stat is false\n"); + spdk_jsonrpc_send_error_response(request, -EPERM, + "RPC not enabled if enable_io_path_stat is false"); + return; + } + + if (spdk_json_decode_object(params, rpc_get_path_stat_decoders, + SPDK_COUNTOF(rpc_get_path_stat_decoders), + &req)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "spdk_json_decode_object failed"); + free_rpc_get_path_stat(&req); + return; + } + + rc = spdk_bdev_open_ext(req.name, false, dummy_bdev_event_cb, NULL, &desc); + if (rc != 0) { + SPDK_ERRLOG("Failed to open bdev '%s': %d\n", req.name, rc); + spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc)); + free_rpc_get_path_stat(&req); + return; + } + + free_rpc_get_path_stat(&req); + + ctx = calloc(1, sizeof(struct rpc_bdev_nvme_path_stat_ctx)); + if (ctx == NULL) { + spdk_bdev_close(desc); + SPDK_ERRLOG("Failed to allocate rpc_bdev_nvme_path_stat_ctx struct\n"); + spdk_jsonrpc_send_error_response(request, -ENOMEM, spdk_strerror(ENOMEM)); + return; + } + + bdev = spdk_bdev_desc_get_bdev(desc); + nbdev = bdev->ctxt; + + pthread_mutex_lock(&nbdev->mutex); + if (nbdev->ref == 0) { + rc = -ENOENT; + goto err; + } + + num_paths = nbdev->ref; + path_stat = calloc(num_paths, sizeof(struct path_stat)); + if (path_stat == NULL) { + rc = -ENOMEM; + SPDK_ERRLOG("Failed to allocate memory for path_stat.\n"); + goto err; + } + + /* store the history stat */ + TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) { + assert(i < num_paths); + path_stat[i].ns = nvme_ns; + path_stat[i].trid = nvme_ns->ctrlr->active_path_id->trid; + + assert(nvme_ns->stat != NULL); + memcpy(&path_stat[i].stat, nvme_ns->stat, sizeof(struct spdk_bdev_io_stat)); + i++; + } + pthread_mutex_unlock(&nbdev->mutex); + + ctx->request = request; + ctx->desc = desc; + ctx->path_stat = path_stat; + ctx->num_paths = num_paths; + + spdk_for_each_channel(nbdev, + rpc_bdev_nvme_path_stat_per_channel, + ctx, + rpc_bdev_nvme_path_stat_done); + return; + +err: + pthread_mutex_unlock(&nbdev->mutex); + spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc)); + spdk_bdev_close(desc); + free(ctx); +} +SPDK_RPC_REGISTER("bdev_nvme_get_path_iostat", rpc_bdev_nvme_get_path_iostat, + SPDK_RPC_RUNTIME) diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index 3bbaba911..4f4567557 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -967,6 +967,20 @@ def bdev_nvme_set_multipath_policy(client, name, policy, selector, rr_min_io): return client.call('bdev_nvme_set_multipath_policy', params) +def bdev_nvme_get_path_iostat(client, name): + """Get I/O statistics for IO paths of the block device. + + Args: + name: bdev name to query + + Returns: + I/O statistics for IO paths of the requested block device. + """ + params = {'name': name} + + return client.call('bdev_nvme_get_path_iostat', params) + + def bdev_nvme_cuse_register(client, name): """Register CUSE devices on NVMe controller. diff --git a/scripts/rpc.py b/scripts/rpc.py index b84fc6881..958f04acc 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -876,6 +876,16 @@ if __name__ == "__main__": p.add_argument('-r', '--rr-min-io', help='Number of IO to route to a path before switching to another for round-robin', required=False) p.set_defaults(func=bdev_nvme_set_multipath_policy) + def bdev_nvme_get_path_iostat(args): + print_dict(rpc.bdev.bdev_nvme_get_path_iostat(args.client, + name=args.name)) + + p = subparsers.add_parser('bdev_nvme_get_path_iostat', + help="""Display current I/O statistics of all the IO paths of the blockdev. It can be + called when io_path_stat is true.""") + p.add_argument('-b', '--name', help="Name of the Blockdev. Example: NVMe0n1", required=True) + p.set_defaults(func=bdev_nvme_get_path_iostat) + def bdev_nvme_cuse_register(args): rpc.bdev.bdev_nvme_cuse_register(args.client, name=args.name)