bdev/nvme: Add bdev_nvme_get_io_paths RPC to monitor I/O path states

Add an new RPC bdev_nvme_get_io_paths to query all active I/O paths.

One io_path belongs to One nvme_bdev_channel.
Each nvme_bdev_channel is associated with one nvme_bdev.

If the RPC bdev_nvme_get_io_paths has a bdev name as a parameter
it can use spdk_for_each_channel() simply for the corresponding
nvme_bdev.

However, users will want to know I/O paths of all nvme_bdevs like
the RPC bdev_get_bdevs.

One io_path has one nvme_qpair. One nvme_qpair belongs to one
nvme_poll_group. By relying on these relationships, the RPC
bdev_nvme_get_io_paths traverses all nvme_poll_groups by using
spdk_for_each_channel() to g_bdev_nvme_ctrlrs.

The RPC bdev_nvme_get_io_paths has two modes, display all or
the specified NVMe bdev's active I/O paths.

The specified bdev name is used just for comparison and empty
array is returned if no matched io_path is found.

Signed-off-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I4a0dbf3ef7aaa9a7b7345fc03dc493cc6d37bc99
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12146
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CHANGELOG.md

@@ -21,6 +21,8 @@ A new API `spdk_bdev_unregister_by_name` was added to handle race conditions cor
 New APIs, `spdk_for_each_bdev` and `spdk_for_each_bdev_leaf`, were added to provide iteration
 safe for race conditions.
 
+A new RPC `bdev_nvme_get_io_paths` was added to get all active I/O paths.
+
 ### idxd
 
 A new parameter `flags` was added to all low level submission and preparation

doc/jsonrpc.md

@@ -3344,6 +3344,56 @@ Example response:
 }
 ~~~
 
+### bdev_nvme_get_io_paths {#rpc_bdev_nvme_get_io_paths}
+
+Display all or the specified NVMe bdev's active I/O paths.
+
+#### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+name                    | Optional | string      | Name of the NVMe bdev
+
+#### Example
+
+Example request:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "method": "bdev_nvme_get_io_paths",
+  "id": 1,
+  "params": {
+    "name": "Nvme0n1"
+  }
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": {
+    "poll_groups": [
+      {
+        "thread": "app_thread",
+        "io_paths": [
+          {
+            "bdev_name": "Nvme0n1",
+            "cntlid": 0,
+            "current": true,
+            "connected": true,
+            "accessible": true
+          }
+        ]
+      }
+    ]
+  }
+}
+~~~
+
 ### bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register}
 
 Register CUSE device on NVMe controller.

module/bdev/nvme/bdev_nvme.c

@@ -6160,4 +6160,28 @@ bdev_nvme_get_ctrlr(struct spdk_bdev *bdev)
 	return nvme_ns->ctrlr->ctrlr;
 }
 
+void
+nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path)
+{
+	struct nvme_ns *nvme_ns = io_path->nvme_ns;
+	struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;
+	const struct spdk_nvme_ctrlr_data *cdata;
+
+	spdk_json_write_object_begin(w);
+
+	spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name);
+
+	cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
+
+	spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid);
+
+	spdk_json_write_named_bool(w, "current", io_path == io_path->nbdev_ch->current_io_path);
+
+	spdk_json_write_named_bool(w, "connected", nvme_io_path_is_connected(io_path));
+
+	spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns));
+
+	spdk_json_write_object_end(w);
+}
+
 SPDK_LOG_REGISTER_COMPONENT(bdev_nvme)

module/bdev/nvme/bdev_nvme.h

@@ -223,6 +223,8 @@ struct nvme_poll_group {
 	TAILQ_HEAD(, nvme_qpair)		qpair_list;
 };
 
+void nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path);
+
 struct nvme_ctrlr *nvme_ctrlr_get_by_name(const char *name);
 
 struct nvme_bdev_ctrlr *nvme_bdev_ctrlr_get_by_name(const char *name);

module/bdev/nvme/bdev_nvme_rpc.c

@@ -2017,3 +2017,112 @@ cleanup:
 }
 SPDK_RPC_REGISTER("bdev_nvme_remove_error_injection", rpc_bdev_nvme_remove_error_injection,
 		  SPDK_RPC_RUNTIME)
+
+struct rpc_get_io_paths {
+	char *name;
+};
+
+static void
+free_rpc_get_io_paths(struct rpc_get_io_paths *r)
+{
+	free(r->name);
+}
+
+static const struct spdk_json_object_decoder rpc_get_io_paths_decoders[] = {
+	{"name", offsetof(struct rpc_get_io_paths, name), spdk_json_decode_string, true},
+};
+
+struct rpc_get_io_paths_ctx {
+	struct rpc_get_io_paths req;
+	struct spdk_jsonrpc_request *request;
+	struct spdk_json_write_ctx *w;
+};
+
+static void
+rpc_bdev_nvme_get_io_paths_done(struct spdk_io_channel_iter *i, int status)
+{
+	struct rpc_get_io_paths_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
+
+	spdk_json_write_array_end(ctx->w);
+
+	spdk_json_write_object_end(ctx->w);
+
+	spdk_jsonrpc_end_result(ctx->request, ctx->w);
+
+	free_rpc_get_io_paths(&ctx->req);
+	free(ctx);
+}
+
+static void
+_rpc_bdev_nvme_get_io_paths(struct spdk_io_channel_iter *i)
+{
+	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
+	struct nvme_poll_group *group = spdk_io_channel_get_ctx(_ch);
+	struct rpc_get_io_paths_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
+	struct nvme_qpair *qpair;
+	struct nvme_io_path *io_path;
+	struct nvme_bdev *nbdev;
+
+	spdk_json_write_object_begin(ctx->w);
+
+	spdk_json_write_named_string(ctx->w, "thread", spdk_thread_get_name(spdk_get_thread()));
+
+	spdk_json_write_named_array_begin(ctx->w, "io_paths");
+
+	TAILQ_FOREACH(qpair, &group->qpair_list, tailq) {
+		TAILQ_FOREACH(io_path, &qpair->io_path_list, tailq) {
+			nbdev = io_path->nvme_ns->bdev;
+
+			if (ctx->req.name != NULL &&
+			    strcmp(ctx->req.name, nbdev->disk.name) != 0) {
+				continue;
+			}
+
+			nvme_io_path_info_json(ctx->w, io_path);
+		}
+	}
+
+	spdk_json_write_array_end(ctx->w);
+
+	spdk_json_write_object_end(ctx->w);
+
+	spdk_for_each_channel_continue(i, 0);
+}
+
+static void
+rpc_bdev_nvme_get_io_paths(struct spdk_jsonrpc_request *request,
+			   const struct spdk_json_val *params)
+{
+	struct rpc_get_io_paths_ctx *ctx;
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (ctx == NULL) {
+		spdk_jsonrpc_send_error_response(request, -ENOMEM, spdk_strerror(ENOMEM));
+		return;
+	}
+
+	if (params != NULL &&
+	    spdk_json_decode_object(params, rpc_get_io_paths_decoders,
+				    SPDK_COUNTOF(rpc_get_io_paths_decoders),
+				    &ctx->req)) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "bdev_nvme_get_io_paths requires no parameters");
+
+		free_rpc_get_io_paths(&ctx->req);
+		free(ctx);
+		return;
+	}
+
+	ctx->request = request;
+	ctx->w = spdk_jsonrpc_begin_result(request);
+
+	spdk_json_write_object_begin(ctx->w);
+
+	spdk_json_write_named_array_begin(ctx->w, "poll_groups");
+
+	spdk_for_each_channel(&g_nvme_bdev_ctrlrs,
+			      _rpc_bdev_nvme_get_io_paths,
+			      ctx,
+			      rpc_bdev_nvme_get_io_paths_done);
+}
+SPDK_RPC_REGISTER("bdev_nvme_get_io_paths", rpc_bdev_nvme_get_io_paths, SPDK_RPC_RUNTIME)

python/spdk/rpc/bdev.py

@@ -811,6 +811,21 @@ def bdev_nvme_stop_discovery(client, name):
     return client.call('bdev_nvme_stop_discovery', params)
 
 
+def bdev_nvme_get_io_paths(client, name):
+    """Display all or the specified NVMe bdev's active I/O paths
+
+    Args:
+        name: Name of the NVMe bdev (optional)
+
+    Returns:
+        List of active I/O paths
+    """
+    params = {}
+    if name:
+        params['name'] = name
+    return client.call('bdev_nvme_get_io_paths', params)
+
+
 def bdev_nvme_cuse_register(client, name):
     """Register CUSE devices on NVMe controller.
 

scripts/rpc.py

@@ -761,6 +761,13 @@ if __name__ == "__main__":
     p.add_argument('-b', '--name', help="Name of the service to stop", required=True)
     p.set_defaults(func=bdev_nvme_stop_discovery)
 
+    def bdev_nvme_get_io_paths(args):
+        print_dict(rpc.bdev.bdev_nvme_get_io_paths(args.client, name=args.name))
+
+    p = subparsers.add_parser('bdev_nvme_get_io_paths', help='Display active I/O paths')
+    p.add_argument('-n', '--name', help="Name of the NVMe bdev", required=False)
+    p.set_defaults(func=bdev_nvme_get_io_paths)
+
     def bdev_nvme_cuse_register(args):
         rpc.bdev.bdev_nvme_cuse_register(args.client,
                                          name=args.name)