nvmf: Add rpc to add listeners to subsystems

The construct_nvmf_subsystem method's "listen_addresses" parameter is
now optional, and new listen addresses may be configured at runtime
using the "nvmf_subsystem_add_listener" method.

Change-Id: Ie0217c5d112e278cc0491a561753f50ed877d842
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/395556
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

app/nvmf_tgt/nvmf_rpc.c

@@ -194,6 +194,15 @@ decode_rpc_listen_address(const struct spdk_json_val *val, void *out)
 	return 0;
 }
 
+static void
+free_rpc_listen_address(struct rpc_listen_address *r)
+{
+	free(r->transport);
+	free(r->adrfam);
+	free(r->traddr);
+	free(r->trsvcid);
+}
+
 static int
 decode_rpc_listen_addresses(const struct spdk_json_val *val, void *out)
 {
@@ -293,10 +302,7 @@ free_rpc_listen_addresses(struct rpc_listen_addresses *r)
 	size_t i;
 
 	for (i = 0; i < r->num_listen_address; i++) {
-		free(r->addresses[i].transport);
+		free_rpc_listen_address(&r->addresses[i]);
-		free(r->addresses[i].adrfam);
-		free(r->addresses[i].traddr);
-		free(r->addresses[i].trsvcid);
 	}
 }
 
@@ -353,7 +359,7 @@ static const struct spdk_json_object_decoder rpc_subsystem_decoders[] = {
 	{"core", offsetof(struct rpc_subsystem, core), spdk_json_decode_int32, true},
 	{"mode", offsetof(struct rpc_subsystem, mode), spdk_json_decode_string, true},
 	{"nqn", offsetof(struct rpc_subsystem, nqn), spdk_json_decode_string},
-	{"listen_addresses", offsetof(struct rpc_subsystem, listen_addresses), decode_rpc_listen_addresses},
+	{"listen_addresses", offsetof(struct rpc_subsystem, listen_addresses), decode_rpc_listen_addresses, true},
 	{"hosts", offsetof(struct rpc_subsystem, hosts), decode_rpc_hosts, true},
 	{"allow_any_host", offsetof(struct rpc_subsystem, allow_any_host), spdk_json_decode_bool, true},
 	{"serial_number", offsetof(struct rpc_subsystem, serial_number), spdk_json_decode_string, true},
@@ -493,3 +499,167 @@ invalid:
 	free_rpc_delete_subsystem(&req);
 }
 SPDK_RPC_REGISTER("delete_nvmf_subsystem", spdk_rpc_delete_nvmf_subsystem)
+
+struct nvmf_rpc_listener_ctx {
+	char				*subnqn;
+	struct rpc_listen_address	address;
+
+	struct spdk_jsonrpc_request	*request;
+	struct spdk_nvme_transport_id	trid;
+	bool				response_sent;
+};
+
+static const struct spdk_json_object_decoder nvmf_rpc_listener_decoder[] = {
+	{"subnqn", offsetof(struct nvmf_rpc_listener_ctx, subnqn), spdk_json_decode_string},
+	{"listen_address", offsetof(struct nvmf_rpc_listener_ctx, address), decode_rpc_listen_address},
+};
+
+static void
+nvmf_rpc_listener_ctx_free(struct nvmf_rpc_listener_ctx *ctx)
+{
+	free(ctx->subnqn);
+	free_rpc_listen_address(&ctx->address);
+	free(ctx);
+}
+
+static void
+nvmf_rpc_listen_resumed(struct spdk_nvmf_subsystem *subsystem,
+			void *cb_arg, int status)
+{
+	struct nvmf_rpc_listener_ctx *ctx = cb_arg;
+	struct spdk_jsonrpc_request *request;
+	struct spdk_json_write_ctx *w;
+
+	request = ctx->request;
+	if (ctx->response_sent) {
+		/* If an error occurred, the response has already been sent. */
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+
+	nvmf_rpc_listener_ctx_free(ctx);
+
+	w = spdk_jsonrpc_begin_result(request);
+	if (w == NULL) {
+		return;
+	}
+
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+}
+
+static void
+nvmf_rpc_listen_paused(struct spdk_nvmf_subsystem *subsystem,
+		       void *cb_arg, int status)
+{
+	struct nvmf_rpc_listener_ctx *ctx = cb_arg;
+
+	if (spdk_nvmf_tgt_listen(g_tgt.tgt, &ctx->trid)) {
+		SPDK_ERRLOG("Unable to add listener.\n");
+		goto invalid;
+	}
+
+	if (spdk_nvmf_subsystem_add_listener(subsystem, &ctx->trid)) {
+		goto invalid;
+	}
+
+	if (spdk_nvmf_subsystem_resume(subsystem, nvmf_rpc_listen_resumed, ctx)) {
+		spdk_jsonrpc_send_error_response(ctx->request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "Internal error");
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+
+	return;
+
+invalid:
+	spdk_jsonrpc_send_error_response(ctx->request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+					 "Invalid parameters");
+	ctx->response_sent = true;
+	if (spdk_nvmf_subsystem_resume(subsystem, nvmf_rpc_listen_resumed, ctx)) {
+		SPDK_ERRLOG("Failed to resume subsystem\n");
+		/* Can't really do anything to recover here - subsystem will remain paused. */
+	}
+}
+
+static void
+nvmf_rpc_subsystem_add_listener(struct spdk_jsonrpc_request *request,
+				const struct spdk_json_val *params)
+{
+	struct nvmf_rpc_listener_ctx *ctx;
+	struct spdk_nvmf_subsystem *subsystem;
+	size_t len;
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (!ctx) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "Out of memory");
+		return;
+	}
+
+	ctx->request = request;
+
+	if (spdk_json_decode_object(params, nvmf_rpc_listener_decoder,
+				    SPDK_COUNTOF(nvmf_rpc_listener_decoder),
+				    ctx)) {
+		SPDK_ERRLOG("spdk_json_decode_object failed\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+
+	subsystem = spdk_nvmf_tgt_find_subsystem(g_tgt.tgt, ctx->subnqn);
+	if (!subsystem) {
+		SPDK_ERRLOG("Unable to find subsystem with NQN %s\n", ctx->subnqn);
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+
+	if (spdk_nvme_transport_id_parse_trtype(&ctx->trid.trtype, ctx->address.transport)) {
+		SPDK_ERRLOG("Invalid transport type: %s\n", ctx->address.transport);
+		spdk_jsonrpc_send_error_response(ctx->request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "Invalid parameters");
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+
+	if (ctx->address.adrfam) {
+		if (spdk_nvme_transport_id_parse_adrfam(&ctx->trid.adrfam, ctx->address.adrfam)) {
+			SPDK_ERRLOG("Invalid adrfam: %s\n", ctx->address.adrfam);
+			spdk_jsonrpc_send_error_response(ctx->request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+							 "Invalid parameters");
+			nvmf_rpc_listener_ctx_free(ctx);
+			return;
+		}
+	} else {
+		ctx->trid.adrfam = SPDK_NVMF_ADRFAM_IPV4;
+	}
+
+	len = strlen(ctx->address.traddr);
+	if (len > sizeof(ctx->trid.traddr) - 1) {
+		SPDK_ERRLOG("Transport address longer than %zu characters: %s\n",
+			    sizeof(ctx->trid.traddr) - 1, ctx->address.traddr);
+		spdk_jsonrpc_send_error_response(ctx->request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "Invalid parameters");
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+	memcpy(ctx->trid.traddr, ctx->address.traddr, len + 1);
+
+	len = strlen(ctx->address.trsvcid);
+	if (len > sizeof(ctx->trid.trsvcid) - 1) {
+		SPDK_ERRLOG("Transport service id longer than %zu characters: %s\n",
+			    sizeof(ctx->trid.trsvcid) - 1, ctx->address.trsvcid);
+		spdk_jsonrpc_send_error_response(ctx->request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "Invalid parameters");
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+	memcpy(ctx->trid.trsvcid, ctx->address.trsvcid, len + 1);
+
+	if (spdk_nvmf_subsystem_pause(subsystem, nvmf_rpc_listen_paused, ctx)) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "Internal error");
+		nvmf_rpc_listener_ctx_free(ctx);
+		return;
+	}
+}
+SPDK_RPC_REGISTER("nvmf_subsystem_add_listener", nvmf_rpc_subsystem_add_listener);

doc/jsonrpc.md

@@ -242,7 +242,7 @@ Name                    | Optional | Type        | Description
 ----------------------- | -------- | ----------- | -----------
 core                    | Optional | number      | Core to run the subsystem's poller on. Default: Automatically assign a core.
 nqn                     | Required | string      | Subsystem NQN
-listen_addresses        | Required | array       | Array of @ref rpc_construct_nvmf_subsystem_listen_address objects
+listen_addresses        | Optional | array       | Array of @ref rpc_construct_nvmf_subsystem_listen_address objects
 hosts                   | Optional | array       | Array of strings containing allowed host NQNs. Default: No hosts allowed.
 allow_any_host          | Optional | boolean     | Allow any host (`true`) or enforce allowed host whitelist (`false`). Default: `false`.
 serial_number           | Required | string      | Serial number of virtual controller
@@ -342,3 +342,45 @@ Example response:
   "result": true
 }
 ~~~
+
+## nvmf_subsystem_add_listener  method {#rpc_nvmf_subsystem_add_listener}
+
+Add a new listen address to an NVMe-oF subsystem.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+subnqn                  | Required | string      | Subsystem NQN
+listen_address          | Required | object      | @ref rpc_construct_nvmf_subsystem_listen_address object
+
+### Example
+
+Example request:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "nvmf_subsystem_add_listener",
+  "params": {
+    "subnqn": "nqn.2016-06.io.spdk:cnode1",
+    "listen_address": {
+      "trtype": "RDMA",
+      "adrfam": "IPv4",
+      "traddr": "192.168.0.123",
+      "trsvcid: "4420"
+    }
+  }
+}
+~~~
+
+Example response:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~

scripts/rpc.py

@@ -337,6 +337,14 @@ if __name__ == "__main__":
                    help='subsystem nqn to be deleted. Example: nqn.2016-06.io.spdk:cnode1.')
     p.set_defaults(func=rpc.nvmf.delete_nvmf_subsystem)
 
+    p = subparsers.add_parser('nvmf_subsystem_add_listener', help='Add a listener to an NVMe-oF subsystem')
+    p.add_argument('nqn', help='NVMe-oF subsystem NQN')
+    p.add_argument('-t', '--trtype', help='NVMe-oF transport type: e.g., rdma', required=True)
+    p.add_argument('-a', '--traddr', help='NVMe-oF transport address: e.g., an ip address', required=True)
+    p.add_argument('-f', '--adrfam', help='NVMe-oF transport adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
+    p.add_argument('-s', '--trsvcid', help='NVMe-oF transport service id: e.g., a port number')
+    p.set_defaults(func=rpc.nvmf.nvmf_subsystem_add_listener)
+
     # pmem
     p = subparsers.add_parser('create_pmem_pool', help='Create pmem pool')
     p.add_argument('pmem_file', help='Path to pmemblk pool file')

scripts/rpc/nvmf.py

@@ -6,15 +6,15 @@ def get_nvmf_subsystems(args):
 
 
 def construct_nvmf_subsystem(args):
-    listen_addresses = [dict(u.split(":") for u in a.split(" "))
-                        for a in args.listen.split(",")]
-
     params = {
         'nqn': args.nqn,
-        'listen_addresses': listen_addresses,
         'serial_number': args.serial_number,
     }
 
+    if args.listen:
+        params['listen_addresses'] = [dict(u.split(":") for u in a.split(" "))
+                                      for a in args.listen.split(",")]
+
     if args.hosts:
         hosts = []
         for u in args.hosts.strip().split(" "):
@@ -44,6 +44,20 @@ def construct_nvmf_subsystem(args):
     args.client.call('construct_nvmf_subsystem', params)
 
 
+def nvmf_subsystem_add_listener(args):
+    listen_address = {'trtype': args.trtype,
+                      'traddr': args.traddr,
+                      'trsvcid': args.trsvcid}
+
+    if args.adrfam:
+        listen_address['adrfam'] = args.adrfam
+
+    params = {'subnqn': args.nqn,
+              'listen_address': listen_address}
+
+    args.client.call('nvmf_subsystem_add_listener', params)
+
+
 def delete_nvmf_subsystem(args):
     params = {'nqn': args.subsystem_nqn}
     args.client.call('delete_nvmf_subsystem', params)

test/nvmf/host/identify.sh

@@ -31,7 +31,8 @@ timing_exit start_nvmf_tgt
 
 bdevs="$bdevs $($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
 
-$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 "trtype:RDMA traddr:$NVMF_FIRST_TARGET_IP trsvcid:4420" '' -a -s SPDK00000000000001 -n "$bdevs"
+$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 '' '' -a -s SPDK00000000000001 -n "$bdevs"
+$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t RDMA -a $NVMF_FIRST_TARGET_IP -s 4420
 
 $rootdir/examples/nvme/identify/identify -r "\
         trtype:RDMA \

test/nvmf/nvme_cli/nvme_cli.sh

@@ -34,7 +34,8 @@ bdevs="$bdevs $($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SI
 
 modprobe -v nvme-rdma
 
-$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 "trtype:RDMA traddr:$NVMF_FIRST_TARGET_IP trsvcid:4420" '' -a -s SPDK00000000000001 -n "$bdevs"
+$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 '' '' -a -s SPDK00000000000001 -n "$bdevs"
+$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t RDMA -a $NVMF_FIRST_TARGET_IP -s 4420
 
 nvme connect -t rdma -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT"
 

test/nvmf/rpc/rpc.sh

@@ -45,7 +45,8 @@ do
 	j=0
 	for bdev in $bdevs; do
 		let j=j+1
-		$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode$j "trtype:RDMA traddr:$NVMF_FIRST_TARGET_IP trsvcid:4420" '' -a -s SPDK00000000000001 -n "$bdev"
+		$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode$j '' '' -a -s SPDK00000000000001 -n "$bdev"
+		$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$j -t RDMA -a $NVMF_FIRST_TARGET_IP -s 4420
 	done
 
 	n=$j