ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bdev_nvme: add support to display io path stat of the NVMe bdev

Browse Source 
Add RPC bdev_nvme_get_path_iostat to show I/O statistics for IO paths
of the NVMe bdev.

Change-Id: I22e5ad112d5cfa6d96cf246dcd0e511ae71dc839
Signed-off-by: Richael Zhuang <richael.zhuang@arm.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14745
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
This commit is contained in:
Richael Zhuang 2022-09-29 11:59:22 +08:00 committed by Tomasz Zawadzki
parent 57cf89617e
commit b317d8f396
5 changed files with 320 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -69,6 +69,8 @@ were added to process I/O statistics outside the generic bdev layer, especially
Added I/O statistics per I/O path to the NVMe bdev module for NVMe bdev multipath. It can be
enabled by a new option io_path_stat of RPC bdev_nvme_set_options.
Added RPC bdev_nvme_get_path_iostat to get I/O statistics for IO paths of the NVMe bdev.
### event
Added core lock file mechanism to prevent the same CPU cores from being used by multiple

96
doc/jsonrpc.md
View File

@ -4211,6 +4211,102 @@ Example response:
}
~~~
### bdev_nvme_get_path_iostat {#rpc_bdev_nvme_get_path_iostat}
Get I/O statistics for IO paths of the block device. Call RPC bdev_nvme_set_options to set enable_io_path_stat
true before using this RPC.
#### Parameters
Name | Optional | Type | Description
----------------------- | -------- | ----------- | -----------
name | Required | string | Name of the NVMe bdev
#### Example
Example request:
~~~json
{
"jsonrpc": "2.0",
"method": "bdev_nvme_get_path_iostat",
"id": 1,
"params": {
"name": "NVMe0n1"
}
}
~~~
Example response:
~~~json
{
"jsonrpc": "2.0",
"id": 1,
"result": {
"name": "NVMe0n1",
"stats": [
{
"trid": {
"trtype": "TCP",
"adrfam": "IPv4",
"traddr": "10.169.204.201",
"trsvcid": "4420",
"subnqn": "nqn.2016-06.io.spdk:cnode1"
},
"stat": {
"bytes_read": 676691968,
"num_read_ops": 165201,
"bytes_written": 0,
"num_write_ops": 0,
"bytes_unmapped": 0,
"num_unmap_ops": 0,
"max_read_latency_ticks": 521487,
"min_read_latency_ticks": 0,
"write_latency_ticks": 0,
"max_write_latency_ticks": 0,
"min_write_latency_ticks": 0,
"unmap_latency_ticks": 0,
"max_unmap_latency_ticks": 0,
"min_unmap_latency_ticks": 0,
"copy_latency_ticks": 0,
"max_copy_latency_ticks": 0,
"min_copy_latency_ticks": 0
}
},
{
"trid": {
"trtype": "TCP",
"adrfam": "IPv4",
"traddr": "8.8.8.6",
"trsvcid": "4420",
"subnqn": "nqn.2016-06.io.spdk:cnode1"
},
"stat": {
"bytes_read": 677138432,
"num_read_ops": 165317,
"bytes_written": 0,
"num_write_ops": 0,
"bytes_unmapped": 0,
"num_unmap_ops": 0,
"max_read_latency_ticks": 108525,
"min_read_latency_ticks": 0,
"write_latency_ticks": 0,
"max_write_latency_ticks": 0,
"min_write_latency_ticks": 0,
"unmap_latency_ticks": 0,
"max_unmap_latency_ticks": 0,
"min_unmap_latency_ticks": 0,
"copy_latency_ticks": 0,
"max_copy_latency_ticks": 0,
"min_copy_latency_ticks": 0
}
}
]
}
}
~~~
### bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register}
Register CUSE device on NVMe controller.

198
module/bdev/nvme/bdev_nvme_rpc.c
View File

@ -2456,3 +2456,201 @@ rpc_bdev_nvme_get_mdns_discovery_info(struct spdk_jsonrpc_request *request,
SPDK_RPC_REGISTER("bdev_nvme_get_mdns_discovery_info", rpc_bdev_nvme_get_mdns_discovery_info,
SPDK_RPC_RUNTIME)
struct rpc_get_path_stat {
char *name;
};
struct path_stat {
struct spdk_bdev_io_stat stat;
struct spdk_nvme_transport_id trid;
struct nvme_ns *ns;
};
struct rpc_bdev_nvme_path_stat_ctx {
struct spdk_jsonrpc_request *request;
struct path_stat *path_stat;
uint32_t num_paths;
struct spdk_bdev_desc *desc;
};
static void
free_rpc_get_path_stat(struct rpc_get_path_stat *req)
{
free(req->name);
}
static const struct spdk_json_object_decoder rpc_get_path_stat_decoders[] = {
{"name", offsetof(struct rpc_get_path_stat, name), spdk_json_decode_string},
};
static void
dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx)
{
}
static void
rpc_bdev_nvme_path_stat_per_channel(struct spdk_io_channel_iter *i)
{
struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);
struct nvme_io_path *io_path;
struct path_stat *path_stat;
uint32_t j;
assert(ctx->num_paths != 0);
for (j = 0; j < ctx->num_paths; j++) {
path_stat = &ctx->path_stat[j];
STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {
if (path_stat->ns == io_path->nvme_ns) {
assert(io_path->stat != NULL);
spdk_bdev_add_io_stat(&path_stat->stat, io_path->stat);
}
}
}
spdk_for_each_channel_continue(i, 0);
}
static void
rpc_bdev_nvme_path_stat_done(struct spdk_io_channel_iter *i, int status)
{
struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);
struct spdk_json_write_ctx *w;
struct path_stat *path_stat;
uint32_t j;
assert(ctx->num_paths != 0);
w = spdk_jsonrpc_begin_result(ctx->request);
spdk_json_write_object_begin(w);
spdk_json_write_named_string(w, "name", nbdev->disk.name);
spdk_json_write_named_array_begin(w, "stats");
for (j = 0; j < ctx->num_paths; j++) {
path_stat = &ctx->path_stat[j];
spdk_json_write_object_begin(w);
spdk_json_write_named_object_begin(w, "trid");
nvme_bdev_dump_trid_json(&path_stat->trid, w);
spdk_json_write_object_end(w);
spdk_json_write_named_object_begin(w, "stat");
spdk_bdev_dump_io_stat_json(&path_stat->stat, w);
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);
}
spdk_json_write_array_end(w);
spdk_json_write_object_end(w);
spdk_jsonrpc_end_result(ctx->request, w);
spdk_bdev_close(ctx->desc);
free(ctx->path_stat);
free(ctx);
}
static void
rpc_bdev_nvme_get_path_iostat(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct rpc_get_path_stat req = {};
struct spdk_bdev_desc *desc = NULL;
struct spdk_bdev *bdev;
struct nvme_bdev *nbdev;
struct nvme_ns *nvme_ns;
struct path_stat *path_stat;
struct rpc_bdev_nvme_path_stat_ctx *ctx;
struct spdk_bdev_nvme_opts opts;
uint32_t num_paths = 0, i = 0;
int rc;
bdev_nvme_get_opts(&opts);
if (!opts.io_path_stat) {
SPDK_ERRLOG("RPC not enabled if enable_io_path_stat is false\n");
spdk_jsonrpc_send_error_response(request, -EPERM,
"RPC not enabled if enable_io_path_stat is false");
return;
}
if (spdk_json_decode_object(params, rpc_get_path_stat_decoders,
SPDK_COUNTOF(rpc_get_path_stat_decoders),
&req)) {
SPDK_ERRLOG("spdk_json_decode_object failed\n");
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
"spdk_json_decode_object failed");
free_rpc_get_path_stat(&req);
return;
}
rc = spdk_bdev_open_ext(req.name, false, dummy_bdev_event_cb, NULL, &desc);
if (rc != 0) {
SPDK_ERRLOG("Failed to open bdev '%s': %d\n", req.name, rc);
spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
free_rpc_get_path_stat(&req);
return;
}
free_rpc_get_path_stat(&req);
ctx = calloc(1, sizeof(struct rpc_bdev_nvme_path_stat_ctx));
if (ctx == NULL) {
spdk_bdev_close(desc);
SPDK_ERRLOG("Failed to allocate rpc_bdev_nvme_path_stat_ctx struct\n");
spdk_jsonrpc_send_error_response(request, -ENOMEM, spdk_strerror(ENOMEM));
return;
}
bdev = spdk_bdev_desc_get_bdev(desc);
nbdev = bdev->ctxt;
pthread_mutex_lock(&nbdev->mutex);
if (nbdev->ref == 0) {
rc = -ENOENT;
goto err;
}
num_paths = nbdev->ref;
path_stat = calloc(num_paths, sizeof(struct path_stat));
if (path_stat == NULL) {
rc = -ENOMEM;
SPDK_ERRLOG("Failed to allocate memory for path_stat.\n");
goto err;
}
/* store the history stat */
TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {
assert(i < num_paths);
path_stat[i].ns = nvme_ns;
path_stat[i].trid = nvme_ns->ctrlr->active_path_id->trid;
assert(nvme_ns->stat != NULL);
memcpy(&path_stat[i].stat, nvme_ns->stat, sizeof(struct spdk_bdev_io_stat));
i++;
}
pthread_mutex_unlock(&nbdev->mutex);
ctx->request = request;
ctx->desc = desc;
ctx->path_stat = path_stat;
ctx->num_paths = num_paths;
spdk_for_each_channel(nbdev,
rpc_bdev_nvme_path_stat_per_channel,
ctx,
rpc_bdev_nvme_path_stat_done);
return;
err:
pthread_mutex_unlock(&nbdev->mutex);
spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));
spdk_bdev_close(desc);
free(ctx);
}
SPDK_RPC_REGISTER("bdev_nvme_get_path_iostat", rpc_bdev_nvme_get_path_iostat,
SPDK_RPC_RUNTIME)

14
python/spdk/rpc/bdev.py
View File

@ -967,6 +967,20 @@ def bdev_nvme_set_multipath_policy(client, name, policy, selector, rr_min_io):
return client.call('bdev_nvme_set_multipath_policy', params)
def bdev_nvme_get_path_iostat(client, name):
"""Get I/O statistics for IO paths of the block device.
Args:
name: bdev name to query
Returns:
I/O statistics for IO paths of the requested block device.
"""
params = {'name': name}
return client.call('bdev_nvme_get_path_iostat', params)
def bdev_nvme_cuse_register(client, name):
"""Register CUSE devices on NVMe controller.

10
scripts/rpc.py
View File

@ -876,6 +876,16 @@ if __name__ == "__main__":
p.add_argument('-r', '--rr-min-io', help='Number of IO to route to a path before switching to another for round-robin', required=False)
p.set_defaults(func=bdev_nvme_set_multipath_policy)
def bdev_nvme_get_path_iostat(args):
print_dict(rpc.bdev.bdev_nvme_get_path_iostat(args.client,
name=args.name))
p = subparsers.add_parser('bdev_nvme_get_path_iostat',
help="""Display current I/O statistics of all the IO paths of the blockdev. It can be
called when io_path_stat is true.""")
p.add_argument('-b', '--name', help="Name of the Blockdev. Example: NVMe0n1", required=True)
p.set_defaults(func=bdev_nvme_get_path_iostat)
def bdev_nvme_cuse_register(args):
rpc.bdev.bdev_nvme_cuse_register(args.client,
name=args.name)