hotplug: use RPC instead of reading hotplug logs

A new RPC perform_tests has been added.
This request will help us avoid reading hotplug generated
logs, and instead we can stop hotplug right before
it begins generating IO, and resume it when the test
scripts are ready.

Additionally a new command line option has been added
to the hotplug application "--wait-for-rpc", which
indicates that hotplug should wait for
perform_tests RPC before starting its IO.

Change-Id: I71ca148201854ac155cc2a61171a4fb5fc427a19
Signed-off-by: Krzysztof Karas <krzysztof.karas@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13962
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

examples/nvme/hotplug/hotplug.c

@@ -10,6 +10,9 @@
 #include "spdk/string.h"
 #include "spdk/util.h"
 #include "spdk/log.h"
+#include "spdk/rpc.h"
+
+static const char *g_rpc_addr = "/var/tmp/spdk.sock";
 
 struct dev_ctx {
 	TAILQ_ENTRY(dev_ctx)	tailq;
@@ -49,6 +52,9 @@ static const char *g_iova_mode = NULL;
 static uint64_t g_timeout_in_us = SPDK_SEC_TO_USEC;
 static struct spdk_nvme_detach_ctx *g_detach_ctx;
 
+static bool g_wait_for_rpc = false;
+static bool g_rpc_received = false;
+
 static void task_complete(struct perf_task *task);
 
 static void timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
@@ -402,24 +408,34 @@ usage(char *program_name)
 	printf("\t[-l log level]\n");
 	printf("\t Available log levels:\n");
 	printf("\t  disabled, error, warning, notice, info, debug\n");
+	printf("\t[--wait-for-rpc wait for RPC perform_tests\n");
+	printf("\t  to proceed with starting IO on NVMe disks]\n");
 }
 
+static const struct option g_wait_option[] = {
+#define WAIT_FOR_RPC_OPT_IDX	257
+	{"wait-for-rpc", no_argument, NULL, WAIT_FOR_RPC_OPT_IDX},
+};
+
 static int
 parse_args(int argc, char **argv)
 {
-	int op;
+	int op, opt_idx;
 	long int val;
 
 	/* default value */
 	g_time_in_sec = 0;
 
-	while ((op = getopt(argc, argv, "c:i:l:m:n:r:t:")) != -1) {
+	while ((op = getopt_long(argc, argv, "c:i:l:m:n:r:t:", g_wait_option, &opt_idx)) != -1) {
 		if (op == '?') {
 			usage(argv[0]);
 			return 1;
 		}
 
 		switch (op) {
+		case WAIT_FOR_RPC_OPT_IDX:
+			g_wait_for_rpc = true;
+			break;
 		case 'c':
 		case 'i':
 		case 'n':
@@ -498,6 +514,39 @@ register_controllers(void)
 	return 0;
 }
 
+/* Hotplug RPC */
+static void
+rpc_perform_tests(struct spdk_jsonrpc_request *request,
+		  const struct spdk_json_val *params)
+{
+	if (params) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "'perform_tests' requires no arguments");
+		return;
+	}
+
+	spdk_jsonrpc_send_bool_response(request, true);
+
+	g_rpc_received = true;
+}
+SPDK_RPC_REGISTER("perform_tests", rpc_perform_tests, SPDK_RPC_RUNTIME);
+
+static void
+wait_for_rpc_call(void)
+{
+	fprintf(stderr,
+		"Listening for perform_tests to start the application...\n");
+	spdk_rpc_listen(g_rpc_addr);
+	spdk_rpc_set_state(SPDK_RPC_RUNTIME);
+
+	while (!g_rpc_received) {
+		spdk_rpc_accept();
+	}
+	/* Run spdk_rpc_accept() one more time to trigger
+	 * spdk_jsonrpv_server_poll() and send the RPC response. */
+	spdk_rpc_accept();
+}
+
 int
 main(int argc, char **argv)
 {
@@ -531,6 +580,10 @@ main(int argc, char **argv)
 		goto cleanup;
 	}
 
+	if (g_wait_for_rpc) {
+		wait_for_rpc_call();
+	}
+
 	fprintf(stderr, "Initialization complete. Starting I/O...\n");
 	io_loop();
 

examples/nvme/hotplug/hotplug_plugin.py

@@ -0,0 +1,11 @@
+from spdk.rpc.client import print_json
+
+
+def perform_tests(args):
+    print_json(args.client.call('perform_tests'))
+
+
+def spdk_rpc_plugin_initialize(subparsers):
+    p = subparsers.add_parser('perform_tests',
+                              help='Returns true when hotplug apps starts running IO')
+    p.set_defaults(func=perform_tests)

scripts/check_format.sh

@@ -614,7 +614,7 @@ function check_json_rpc() {
 			echo "Missing JSON-RPC documentation for ${rpc}"
 			rc=1
 		fi
-	done < <(git grep -h -E "^SPDK_RPC_REGISTER\(" ':!test/*')
+	done < <(git grep -h -E "^SPDK_RPC_REGISTER\(" ':!test/*' ':!examples/nvme/hotplug/*')
 
 	if [ $rc -eq 0 ]; then
 		echo " OK"

test/nvme/hw_hotplug.sh

@@ -5,6 +5,9 @@ testdir=$(readlink -f $(dirname $0))
 rootdir=$(readlink -f $testdir/../..)
 source $rootdir/test/common/autotest_common.sh
 
+export PYTHONPATH="$rootdir/examples/nvme/hotplug/"
+rpc_py=$rootdir/scripts/rpc.py
+
 function beetle_ssh() {
 	ssh_opts="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
 	if [[ -n $BEETLE_SSH_KEY ]]; then
@@ -69,25 +72,14 @@ if [ "$driver" = "uio_pci_generic" ]; then
 	mode="-m pa"
 fi
 
-exec {log}> >(tee -a "$testdir/log.txt")
-exec >&$log 2>&1
-
-"$SPDK_EXAMPLE_DIR/hotplug" -i 0 -t 100 -n $((2 * nvme_count)) -r $((2 * nvme_count)) $mode &
+"$SPDK_EXAMPLE_DIR/hotplug" -i 0 -t 100 -n $((2 * nvme_count)) -r $((2 * nvme_count)) \
+	$mode --wait-for-rpc &
 hotplug_pid=$!
 
-trap 'killprocess $hotplug_pid; restore_device; rm $testdir/log.txt; exit 1' SIGINT SIGTERM EXIT
+trap 'killprocess $hotplug_pid; restore_device; exit 1' SIGINT SIGTERM EXIT
 
-i=0
-while ! grep -q "Starting I/O" $testdir/log.txt; do
-	[ $i -lt 20 ] || break
-	i=$((i + 1))
-	sleep 1
-done
-
-if ! grep -q "Starting I/O" $testdir/log.txt; then
-	rm $testdir/log.txt
-	exit 1
-fi
+waitforlisten $hotplug_pid
+$rpc_py --plugin hotplug_plugin perform_tests
 
 # Add and remove NVMe with delays between to give some time for IO to proceed
 remove_device
@@ -103,14 +95,11 @@ timing_enter wait_for_example
 
 if ! wait $hotplug_pid; then
 	echo "Hotplug example returned error!"
-	rm $testdir/log.txt
 	exit 1
 fi
 
 timing_exit wait_for_example
 
-rm $testdir/log.txt
-
 trap - SIGINT SIGTERM EXIT
 
 restore_device