bdev_nvme: add support to display io path stat of the NVMe bdev
Add RPC bdev_nvme_get_path_iostat to show I/O statistics for IO paths of the NVMe bdev. Change-Id: I22e5ad112d5cfa6d96cf246dcd0e511ae71dc839 Signed-off-by: Richael Zhuang <richael.zhuang@arm.com> Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14745 Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
This commit is contained in:
Richael Zhuang
Tomasz Zawadzki
parent
57cf89617e
commit
b317d8f396
@ -69,6 +69,8 @@ were added to process I/O statistics outside the generic bdev layer, especially
|
|||||||
Added I/O statistics per I/O path to the NVMe bdev module for NVMe bdev multipath. It can be
|
Added I/O statistics per I/O path to the NVMe bdev module for NVMe bdev multipath. It can be
|
||||||
enabled by a new option io_path_stat of RPC bdev_nvme_set_options.
|
enabled by a new option io_path_stat of RPC bdev_nvme_set_options.
|
||||||
|
|
||||||
|
Added RPC bdev_nvme_get_path_iostat to get I/O statistics for IO paths of the NVMe bdev.
|
||||||
|
|
||||||
### event
|
### event
|
||||||
|
|
||||||
Added core lock file mechanism to prevent the same CPU cores from being used by multiple
|
Added core lock file mechanism to prevent the same CPU cores from being used by multiple
|
||||||
|
|||||||
@ -4211,6 +4211,102 @@ Example response:
|
|||||||
}
|
}
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
|
### bdev_nvme_get_path_iostat {#rpc_bdev_nvme_get_path_iostat}
|
||||||
|
|
||||||
|
Get I/O statistics for IO paths of the block device. Call RPC bdev_nvme_set_options to set enable_io_path_stat
|
||||||
|
true before using this RPC.
|
||||||
|
|
||||||
|
#### Parameters
|
||||||
|
|
||||||
|
Name | Optional | Type | Description
|
||||||
|
----------------------- | -------- | ----------- | -----------
|
||||||
|
name | Required | string | Name of the NVMe bdev
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
Example request:
|
||||||
|
|
||||||
|
~~~json
|
||||||
|
{
|
||||||
|
"jsonrpc": "2.0",
|
||||||
|
"method": "bdev_nvme_get_path_iostat",
|
||||||
|
"id": 1,
|
||||||
|
"params": {
|
||||||
|
"name": "NVMe0n1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Example response:
|
||||||
|
|
||||||
|
~~~json
|
||||||
|
{
|
||||||
|
"jsonrpc": "2.0",
|
||||||
|
"id": 1,
|
||||||
|
"result": {
|
||||||
|
"name": "NVMe0n1",
|
||||||
|
"stats": [
|
||||||
|
{
|
||||||
|
"trid": {
|
||||||
|
"trtype": "TCP",
|
||||||
|
"adrfam": "IPv4",
|
||||||
|
"traddr": "10.169.204.201",
|
||||||
|
"trsvcid": "4420",
|
||||||
|
"subnqn": "nqn.2016-06.io.spdk:cnode1"
|
||||||
|
},
|
||||||
|
"stat": {
|
||||||
|
"bytes_read": 676691968,
|
||||||
|
"num_read_ops": 165201,
|
||||||
|
"bytes_written": 0,
|
||||||
|
"num_write_ops": 0,
|
||||||
|
"bytes_unmapped": 0,
|
||||||
|
"num_unmap_ops": 0,
|
||||||
|
"max_read_latency_ticks": 521487,
|
||||||
|
"min_read_latency_ticks": 0,
|
||||||
|
"write_latency_ticks": 0,
|
||||||
|
"max_write_latency_ticks": 0,
|
||||||
|
"min_write_latency_ticks": 0,
|
||||||
|
"unmap_latency_ticks": 0,
|
||||||
|
"max_unmap_latency_ticks": 0,
|
||||||
|
"min_unmap_latency_ticks": 0,
|
||||||
|
"copy_latency_ticks": 0,
|
||||||
|
"max_copy_latency_ticks": 0,
|
||||||
|
"min_copy_latency_ticks": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"trid": {
|
||||||
|
"trtype": "TCP",
|
||||||
|
"adrfam": "IPv4",
|
||||||
|
"traddr": "8.8.8.6",
|
||||||
|
"trsvcid": "4420",
|
||||||
|
"subnqn": "nqn.2016-06.io.spdk:cnode1"
|
||||||
|
},
|
||||||
|
"stat": {
|
||||||
|
"bytes_read": 677138432,
|
||||||
|
"num_read_ops": 165317,
|
||||||
|
"bytes_written": 0,
|
||||||
|
"num_write_ops": 0,
|
||||||
|
"bytes_unmapped": 0,
|
||||||
|
"num_unmap_ops": 0,
|
||||||
|
"max_read_latency_ticks": 108525,
|
||||||
|
"min_read_latency_ticks": 0,
|
||||||
|
"write_latency_ticks": 0,
|
||||||
|
"max_write_latency_ticks": 0,
|
||||||
|
"min_write_latency_ticks": 0,
|
||||||
|
"unmap_latency_ticks": 0,
|
||||||
|
"max_unmap_latency_ticks": 0,
|
||||||
|
"min_unmap_latency_ticks": 0,
|
||||||
|
"copy_latency_ticks": 0,
|
||||||
|
"max_copy_latency_ticks": 0,
|
||||||
|
"min_copy_latency_ticks": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
~~~
|
||||||
|
|
||||||
### bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register}
|
### bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register}
|
||||||
|
|
||||||
Register CUSE device on NVMe controller.
|
Register CUSE device on NVMe controller.
|
||||||
|
|||||||
@ -2456,3 +2456,201 @@ rpc_bdev_nvme_get_mdns_discovery_info(struct spdk_jsonrpc_request *request,
|
|||||||
|
|
||||||
SPDK_RPC_REGISTER("bdev_nvme_get_mdns_discovery_info", rpc_bdev_nvme_get_mdns_discovery_info,
|
SPDK_RPC_REGISTER("bdev_nvme_get_mdns_discovery_info", rpc_bdev_nvme_get_mdns_discovery_info,
|
||||||
SPDK_RPC_RUNTIME)
|
SPDK_RPC_RUNTIME)
|
||||||
|
|
||||||
|
struct rpc_get_path_stat {
|
||||||
|
char *name;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct path_stat {
|
||||||
|
struct spdk_bdev_io_stat stat;
|
||||||
|
struct spdk_nvme_transport_id trid;
|
||||||
|
struct nvme_ns *ns;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct rpc_bdev_nvme_path_stat_ctx {
|
||||||
|
struct spdk_jsonrpc_request *request;
|
||||||
|
struct path_stat *path_stat;
|
||||||
|
uint32_t num_paths;
|
||||||
|
struct spdk_bdev_desc *desc;
|
||||||
|
};
|
||||||
|
|
||||||
|
static void
|
||||||
|
free_rpc_get_path_stat(struct rpc_get_path_stat *req)
|
||||||
|
{
|
||||||
|
free(req->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
static const struct spdk_json_object_decoder rpc_get_path_stat_decoders[] = {
|
||||||
|
{"name", offsetof(struct rpc_get_path_stat, name), spdk_json_decode_string},
|
||||||
|
};
|
||||||
|
|
||||||
|
static void
|
||||||
|
dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
rpc_bdev_nvme_path_stat_per_channel(struct spdk_io_channel_iter *i)
|
||||||
|
{
|
||||||
|
struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
||||||
|
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
|
||||||
|
struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
|
||||||
|
struct nvme_io_path *io_path;
|
||||||
|
struct path_stat *path_stat;
|
||||||
|
uint32_t j;
|
||||||
|
|
||||||
|
assert(ctx->num_paths != 0);
|
||||||
|
|
||||||
|
for (j = 0; j < ctx->num_paths; j++) {
|
||||||
|
path_stat = &ctx->path_stat[j];
|
||||||
|
|
||||||
|
STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
|
||||||
|
if (path_stat->ns == io_path->nvme_ns) {
|
||||||
|
assert(io_path->stat != NULL);
|
||||||
|
spdk_bdev_add_io_stat(&path_stat->stat, io_path->stat);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spdk_for_each_channel_continue(i, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
rpc_bdev_nvme_path_stat_done(struct spdk_io_channel_iter *i, int status)
|
||||||
|
{
|
||||||
|
struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
||||||
|
struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
|
||||||
|
struct spdk_json_write_ctx *w;
|
||||||
|
struct path_stat *path_stat;
|
||||||
|
uint32_t j;
|
||||||
|
|
||||||
|
assert(ctx->num_paths != 0);
|
||||||
|
|
||||||
|
w = spdk_jsonrpc_begin_result(ctx->request);
|
||||||
|
spdk_json_write_object_begin(w);
|
||||||
|
spdk_json_write_named_string(w, "name", nbdev->disk.name);
|
||||||
|
spdk_json_write_named_array_begin(w, "stats");
|
||||||
|
|
||||||
|
for (j = 0; j < ctx->num_paths; j++) {
|
||||||
|
path_stat = &ctx->path_stat[j];
|
||||||
|
spdk_json_write_object_begin(w);
|
||||||
|
|
||||||
|
spdk_json_write_named_object_begin(w, "trid");
|
||||||
|
nvme_bdev_dump_trid_json(&path_stat->trid, w);
|
||||||
|
spdk_json_write_object_end(w);
|
||||||
|
|
||||||
|
spdk_json_write_named_object_begin(w, "stat");
|
||||||
|
spdk_bdev_dump_io_stat_json(&path_stat->stat, w);
|
||||||
|
spdk_json_write_object_end(w);
|
||||||
|
|
||||||
|
spdk_json_write_object_end(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
spdk_json_write_array_end(w);
|
||||||
|
spdk_json_write_object_end(w);
|
||||||
|
spdk_jsonrpc_end_result(ctx->request, w);
|
||||||
|
|
||||||
|
spdk_bdev_close(ctx->desc);
|
||||||
|
free(ctx->path_stat);
|
||||||
|
free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
rpc_bdev_nvme_get_path_iostat(struct spdk_jsonrpc_request *request,
|
||||||
|
const struct spdk_json_val *params)
|
||||||
|
{
|
||||||
|
struct rpc_get_path_stat req = {};
|
||||||
|
struct spdk_bdev_desc *desc = NULL;
|
||||||
|
struct spdk_bdev *bdev;
|
||||||
|
struct nvme_bdev *nbdev;
|
||||||
|
struct nvme_ns *nvme_ns;
|
||||||
|
struct path_stat *path_stat;
|
||||||
|
struct rpc_bdev_nvme_path_stat_ctx *ctx;
|
||||||
|
struct spdk_bdev_nvme_opts opts;
|
||||||
|
uint32_t num_paths = 0, i = 0;
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
bdev_nvme_get_opts(&opts);
|
||||||
|
if (!opts.io_path_stat) {
|
||||||
|
SPDK_ERRLOG("RPC not enabled if enable_io_path_stat is false\n");
|
||||||
|
spdk_jsonrpc_send_error_response(request, -EPERM,
|
||||||
|
"RPC not enabled if enable_io_path_stat is false");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (spdk_json_decode_object(params, rpc_get_path_stat_decoders,
|
||||||
|
SPDK_COUNTOF(rpc_get_path_stat_decoders),
|
||||||
|
&req)) {
|
||||||
|
SPDK_ERRLOG("spdk_json_decode_object failed\n");
|
||||||
|
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
|
||||||
|
"spdk_json_decode_object failed");
|
||||||
|
free_rpc_get_path_stat(&req);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
rc = spdk_bdev_open_ext(req.name, false, dummy_bdev_event_cb, NULL, &desc);
|
||||||
|
if (rc != 0) {
|
||||||
|
SPDK_ERRLOG("Failed to open bdev '%s': %d\n", req.name, rc);
|
||||||
|
spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
|
||||||
|
free_rpc_get_path_stat(&req);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
free_rpc_get_path_stat(&req);
|
||||||
|
|
||||||
|
ctx = calloc(1, sizeof(struct rpc_bdev_nvme_path_stat_ctx));
|
||||||
|
if (ctx == NULL) {
|
||||||
|
spdk_bdev_close(desc);
|
||||||
|
SPDK_ERRLOG("Failed to allocate rpc_bdev_nvme_path_stat_ctx struct\n");
|
||||||
|
spdk_jsonrpc_send_error_response(request, -ENOMEM, spdk_strerror(ENOMEM));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
bdev = spdk_bdev_desc_get_bdev(desc);
|
||||||
|
nbdev = bdev->ctxt;
|
||||||
|
|
||||||
|
pthread_mutex_lock(&nbdev->mutex);
|
||||||
|
if (nbdev->ref == 0) {
|
||||||
|
rc = -ENOENT;
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
num_paths = nbdev->ref;
|
||||||
|
path_stat = calloc(num_paths, sizeof(struct path_stat));
|
||||||
|
if (path_stat == NULL) {
|
||||||
|
rc = -ENOMEM;
|
||||||
|
SPDK_ERRLOG("Failed to allocate memory for path_stat.\n");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* store the history stat */
|
||||||
|
TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
|
||||||
|
assert(i < num_paths);
|
||||||
|
path_stat[i].ns = nvme_ns;
|
||||||
|
path_stat[i].trid = nvme_ns->ctrlr->active_path_id->trid;
|
||||||
|
|
||||||
|
assert(nvme_ns->stat != NULL);
|
||||||
|
memcpy(&path_stat[i].stat, nvme_ns->stat, sizeof(struct spdk_bdev_io_stat));
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
pthread_mutex_unlock(&nbdev->mutex);
|
||||||
|
|
||||||
|
ctx->request = request;
|
||||||
|
ctx->desc = desc;
|
||||||
|
ctx->path_stat = path_stat;
|
||||||
|
ctx->num_paths = num_paths;
|
||||||
|
|
||||||
|
spdk_for_each_channel(nbdev,
|
||||||
|
rpc_bdev_nvme_path_stat_per_channel,
|
||||||
|
ctx,
|
||||||
|
rpc_bdev_nvme_path_stat_done);
|
||||||
|
return;
|
||||||
|
|
||||||
|
err:
|
||||||
|
pthread_mutex_unlock(&nbdev->mutex);
|
||||||
|
spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
|
||||||
|
spdk_bdev_close(desc);
|
||||||
|
free(ctx);
|
||||||
|
}
|
||||||
|
SPDK_RPC_REGISTER("bdev_nvme_get_path_iostat", rpc_bdev_nvme_get_path_iostat,
|
||||||
|
SPDK_RPC_RUNTIME)
|
||||||
|
|||||||
@ -967,6 +967,20 @@ def bdev_nvme_set_multipath_policy(client, name, policy, selector, rr_min_io):
|
|||||||
return client.call('bdev_nvme_set_multipath_policy', params)
|
return client.call('bdev_nvme_set_multipath_policy', params)
|
||||||
|
|
||||||
|
|
||||||
|
def bdev_nvme_get_path_iostat(client, name):
|
||||||
|
"""Get I/O statistics for IO paths of the block device.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: bdev name to query
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
I/O statistics for IO paths of the requested block device.
|
||||||
|
"""
|
||||||
|
params = {'name': name}
|
||||||
|
|
||||||
|
return client.call('bdev_nvme_get_path_iostat', params)
|
||||||
|
|
||||||
|
|
||||||
def bdev_nvme_cuse_register(client, name):
|
def bdev_nvme_cuse_register(client, name):
|
||||||
"""Register CUSE devices on NVMe controller.
|
"""Register CUSE devices on NVMe controller.
|
||||||
|
|
||||||
|
|||||||
@ -876,6 +876,16 @@ if __name__ == "__main__":
|
|||||||
p.add_argument('-r', '--rr-min-io', help='Number of IO to route to a path before switching to another for round-robin', required=False)
|
p.add_argument('-r', '--rr-min-io', help='Number of IO to route to a path before switching to another for round-robin', required=False)
|
||||||
p.set_defaults(func=bdev_nvme_set_multipath_policy)
|
p.set_defaults(func=bdev_nvme_set_multipath_policy)
|
||||||
|
|
||||||
|
def bdev_nvme_get_path_iostat(args):
|
||||||
|
print_dict(rpc.bdev.bdev_nvme_get_path_iostat(args.client,
|
||||||
|
name=args.name))
|
||||||
|
|
||||||
|
p = subparsers.add_parser('bdev_nvme_get_path_iostat',
|
||||||
|
help="""Display current I/O statistics of all the IO paths of the blockdev. It can be
|
||||||
|
called when io_path_stat is true.""")
|
||||||
|
p.add_argument('-b', '--name', help="Name of the Blockdev. Example: NVMe0n1", required=True)
|
||||||
|
p.set_defaults(func=bdev_nvme_get_path_iostat)
|
||||||
|
|
||||||
def bdev_nvme_cuse_register(args):
|
def bdev_nvme_cuse_register(args):
|
||||||
rpc.bdev.bdev_nvme_cuse_register(args.client,
|
rpc.bdev.bdev_nvme_cuse_register(args.client,
|
||||||
name=args.name)
|
name=args.name)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user