nvmf: add Namespace attribute notice support

Users can use RPC to add/remove a namespace to/from
existing NVMe controller, SPDK NVMeoF target will
generate an asynchronous event as an indication to
host when asynchronous event request is available.

While here, we also set the event with invalid log
identifier, so that the host doesn't need to clear
the event. Users can use Set Feature to disable
such event.

Change-Id: I93c4d752f552d3c86c53e80877aa61c093e167cc
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/398759
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/nvmf/ctrlr.c

@@ -151,7 +151,7 @@ spdk_nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem,
 	ctrlr->max_qpairs_allowed = tgt->opts.max_qpairs_per_ctrlr;
 
 	ctrlr->feat.keep_alive_timer.bits.kato = connect_cmd->kato;
-
+	ctrlr->feat.async_event_configuration.bits.ns_attr_notice = 1;
 	ctrlr->feat.volatile_write_cache.bits.wce = 1;
 
 	/* Subtract 1 for admin queue, 1 for 0's based */
@@ -958,6 +958,13 @@ spdk_nvmf_ctrlr_async_event_request(struct spdk_nvmf_request *req)
 		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
 	}
 
+	if (ctrlr->notice_event.bits.async_event_type ==
+	    SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) {
+		rsp->cdw0 = ctrlr->notice_event.raw;
+		ctrlr->notice_event.raw = 0;
+		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
+	}
+
 	ctrlr->aer_req = req;
 	return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;
 }
@@ -1546,3 +1553,49 @@ spdk_nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req)
 		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
 	}
 }
+
+int
+spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr)
+{
+	struct spdk_nvmf_request *req;
+	struct spdk_nvme_cpl *rsp;
+	union spdk_nvme_async_event_completion event = {0};
+
+	/* Users may disable the event notification */
+	if (!ctrlr->feat.async_event_configuration.bits.ns_attr_notice) {
+		return 0;
+	}
+
+	event.bits.async_event_type = SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE;
+	event.bits.async_event_info = SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED;
+	/* Alternatively, host may request Changed Namespace List log(04h)
+	 * to determine which namespaces have changed. While here, we
+	 * set invalid log page identifier to indicate that host doesn't
+	 * need to send such log page.
+	 */
+	event.bits.log_page_identifier = 0;
+
+	/* If there is no outstanding AER request, queue the event.  Then
+	 * if an AER is later submitted, this event can be sent as a
+	 * response.
+	 */
+	if (!ctrlr->aer_req) {
+		if (ctrlr->notice_event.bits.async_event_type ==
+		    SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) {
+			return 0;
+		}
+
+		ctrlr->notice_event.raw = event.raw;
+		return 0;
+	}
+
+	req = ctrlr->aer_req;
+	rsp = &req->rsp->nvme_cpl;
+
+	rsp->cdw0 = event.raw;
+
+	spdk_nvmf_request_complete(req);
+	ctrlr->aer_req = NULL;
+
+	return 0;
+}

lib/nvmf/nvmf_internal.h

@@ -186,6 +186,7 @@ struct spdk_nvmf_ctrlr {
 	int num_qpairs;
 	int max_qpairs_allowed;
 	struct spdk_nvmf_request *aer_req;
+	union spdk_nvme_async_event_completion notice_event;
 	uint8_t hostid[16];
 
 	TAILQ_ENTRY(spdk_nvmf_ctrlr)		link;
@@ -260,6 +261,7 @@ void spdk_nvmf_subsystem_remove_ctrlr(struct spdk_nvmf_subsystem *subsystem,
 				      struct spdk_nvmf_ctrlr *ctrlr);
 struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem *subsystem,
 		uint16_t cntlid);
+int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr);
 
 static inline struct spdk_nvmf_ns *
 _spdk_nvmf_subsystem_get_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid)

lib/nvmf/subsystem.c

@@ -814,6 +814,7 @@ int
 spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid)
 {
 	struct spdk_nvmf_ns *ns;
+	struct spdk_nvmf_ctrlr *ctrlr, *ctrlr_tmp;
 
 	assert(subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSED ||
 	       subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE);
@@ -838,6 +839,10 @@ spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t ns
 	free(ns);
 	subsystem->num_allocated_nsid--;
 
+	TAILQ_FOREACH_SAFE(ctrlr, &subsystem->ctrlrs, link, ctrlr_tmp) {
+		spdk_nvmf_ctrlr_async_event_ns_notice(ctrlr);
+	}
+
 	return 0;
 }
 
@@ -877,6 +882,7 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
 {
 	struct spdk_nvmf_ns_opts opts;
 	struct spdk_nvmf_ns *ns;
+	struct spdk_nvmf_ctrlr *ctrlr, *ctrlr_tmp;
 	uint32_t i;
 	int rc;
 
@@ -915,11 +921,6 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
 			new_max_nsid = subsystem->max_nsid + 1;
 		}
 
-		if (!TAILQ_EMPTY(&subsystem->ctrlrs)) {
-			SPDK_ERRLOG("Can't extend NSID range with active connections\n");
-			return 0;
-		}
-
 		new_ns_array = realloc(subsystem->ns, sizeof(struct spdk_nvmf_ns *) * new_max_nsid);
 		if (new_ns_array == NULL) {
 			SPDK_ERRLOG("Memory allocation error while resizing namespace array.\n");
@@ -970,6 +971,10 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
 	}
 	subsystem->ns[opts.nsid - 1] = ns;
 
+	TAILQ_FOREACH_SAFE(ctrlr, &subsystem->ctrlrs, link, ctrlr_tmp) {
+		spdk_nvmf_ctrlr_async_event_ns_notice(ctrlr);
+	}
+
 	SPDK_DEBUGLOG(SPDK_LOG_NVMF, "Subsystem %s: bdev %s assigned nsid %" PRIu32 "\n",
 		      spdk_nvmf_subsystem_get_nqn(subsystem),
 		      spdk_bdev_get_name(bdev),

test/nvmf/host/aer.sh

@@ -16,6 +16,11 @@ if [ -z $NVMF_FIRST_TARGET_IP ]; then
 	exit 0
 fi
 
+if check_ip_is_soft_roce $NVMF_FIRST_TARGET_IP; then
+        echo "Bypass AER tests for softRoCE NIC"
+        exit 0
+fi
+
 timing_enter aer
 timing_enter start_nvmf_tgt
 
@@ -27,19 +32,64 @@ trap "killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT
 waitforlisten $nvmfpid
 timing_exit start_nvmf_tgt
 
-bdevs="$bdevs $($rpc_py construct_malloc_bdev 64 512)"
-$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 "trtype:RDMA traddr:$NVMF_FIRST_TARGET_IP trsvcid:4420" '' -a -s SPDK00000000000001 -n "$bdevs"
+modprobe -v nvme-rdma
 
-$rootdir/test/nvme/aer/aer -r "\
-        trtype:RDMA \
-        adrfam:IPv4 \
-        traddr:$NVMF_FIRST_TARGET_IP \
-        trsvcid:$NVMF_PORT \
-        subnqn:nqn.2014-08.org.nvmexpress.discovery"
+$rpc_py construct_malloc_bdev 64 512 --name Malloc0
+$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 "trtype:RDMA traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" '' -a -s SPDK00000000000001 -n Malloc0
+
+# TODO: this aer test tries to invoke an AER completion by setting the temperature
+#threshold to a very low value.  This does not work with emulated controllers
+#though so currently the test is disabled.
+
+#$rootdir/test/nvme/aer/aer -r "\
+#        trtype:RDMA \
+#        adrfam:IPv4 \
+#        traddr:$NVMF_FIRST_TARGET_IP \
+#        trsvcid:$NVMF_PORT \
+#        subnqn:nqn.2014-08.org.nvmexpress.discovery"
+
+# Namespace Attribute Notice Tests with kernel initiator
+nvme connect -t rdma -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT"
+sleep 2
 sync
+
+function get_nvme_name {
+	bdevs=$(lsblk -d | cut -d " " -f 1 | grep "^nvme[0-9]n2") || true
+}
+
+bdevs=""
+get_nvme_name
+
+if [ -n "$bdevs" ]; then
+	echo "Ignore adding Namespace 2 test"
+	$rpc_py delete_bdev Malloc0
+	nvmfcleanup
+	killprocess $nvmfpid
+	exit 0
+fi
+
+# Add a new namespace
+$rpc_py construct_malloc_bdev 128 4096 --name Malloc1
+$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 -n 2
+sleep 3
+sync
+
+bdevs=""
+get_nvme_name
+
+if [ -z "$bdevs" ]; then
+        echo "AER for adding a Namespace test failed"
+	nvmfcleanup
+	killprocess $nvmfpid
+        exit 1
+fi
+
+$rpc_py delete_bdev Malloc0
+$rpc_py delete_bdev Malloc1
 $rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1
 
 trap - SIGINT SIGTERM EXIT
 
+nvmfcleanup
 killprocess $nvmfpid
 timing_exit aer

test/nvmf/nvmf.sh

@@ -35,15 +35,11 @@ fi
 
 timing_enter host
 
-if [ $RUN_NIGHTLY -eq 1 ]; then
-	# TODO: temporarily disabled - temperature AER doesn't fire on emulated controllers
-	#run_test test/nvmf/host/aer.sh
-	true
-fi
 run_test test/nvmf/host/bdevperf.sh
 run_test test/nvmf/host/identify.sh
 run_test test/nvmf/host/perf.sh
 run_test test/nvmf/host/identify_kernel_nvmf.sh
+run_test test/nvmf/host/aer.sh
 run_test test/nvmf/host/fio.sh
 
 timing_exit host

test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c

@@ -186,6 +186,12 @@ spdk_nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group,
 	return 0;
 }
 
+int
+spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr)
+{
+	return 0;
+}
+
 static void
 test_discovery_log(void)
 {

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

@@ -207,6 +207,12 @@ spdk_bdev_get_uuid(const struct spdk_bdev *bdev)
 	return &bdev->uuid;
 }
 
+int
+spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr)
+{
+	return 0;
+}
+
 static void
 test_spdk_nvmf_subsystem_add_ns(void)
 {