sma: wait until SPDK process is responding

Most of the devices need to send RPCs during initialization (e.g. create
a transport), so we need to take care that they're initialized only
after we are certain that the SPDK process is listening.

The mechanisim is similar to the `waitforlisten` function used in our
test scripts - it sends a series of `rpc_get_methods` calls and times
out after a period of time if no response is received.  For now, the
timeout is hardcoded to 60s.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Iddadc04ad4c486d2894bc40e1a899a9d204400fc
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11802
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

scripts/sma.py

@@ -7,12 +7,13 @@ import os
 import signal
 import sys
 import threading
+import time
 import yaml
 
 sys.path.append(os.path.dirname(__file__) + '/../python')
 
-import spdk.sma as sma                      # noqa
+import spdk.sma as sma               # noqa
-from spdk.rpc.client import JSONRPCClient   # noqa
+import spdk.rpc.client as rpcclient  # noqa
 
 
 def parse_config(path):
@@ -47,7 +48,7 @@ def parse_argv():
 
 def get_build_client(sock):
     def build_client():
-        return JSONRPCClient(sock)
+        return rpcclient.JSONRPCClient(sock)
 
     return build_client
 
@@ -74,6 +75,24 @@ def load_plugins(plugins, client):
     return devices
 
 
+def wait_for_listen(client, timeout):
+    start = time.monotonic()
+    while True:
+        try:
+            with client() as _client:
+                _client.call('rpc_get_methods')
+            # If we got here, the process is responding to RPCs
+            break
+        except rpcclient.JSONRPCException:
+            logging.debug('The SPDK process is not responding for {}s'.format(
+                          int(time.monotonic() - start)))
+
+        if time.monotonic() > start + timeout:
+            logging.error('Timed out while waiting for SPDK process to respond')
+            sys.exit(1)
+        time.sleep(1)
+
+
 def run(agent):
     event = threading.Event()
 
@@ -94,6 +113,9 @@ if __name__ == '__main__':
     config = parse_argv()
     client = get_build_client(config['socket'])
 
+    # Wait until the SPDK process starts responding to RPCs
+    wait_for_listen(client, timeout=60.0)
+
     agent = sma.StorageManagementAgent(config['address'], config['port'])
 
     devices = [sma.NvmfTcpDeviceManager(client)]

test/sma/nvmf_tcp.sh

@@ -75,10 +75,6 @@ trap "cleanup; exit 1" SIGINT SIGTERM EXIT
 
 $rootdir/build/bin/spdk_tgt &
 tgtpid=$!
-waitforlisten $tgtpid
-
-# Prepare the target
-rpc_cmd bdev_null_create null0 100 4096
 
 $rootdir/scripts/sma.py -c <(
 	cat <<- EOF
@@ -93,6 +89,9 @@ smapid=$!
 # Wait until the SMA starts listening
 sma_waitforlisten
 
+# Prepare the target
+rpc_cmd bdev_null_create null0 100 4096
+
 # Make sure a TCP transport has been created
 rpc_cmd nvmf_get_transports --trtype tcp
 

test/sma/plugins.sh

@@ -6,6 +6,11 @@ rootdir=$(readlink -f "$testdir/../..")
 source "$rootdir/test/common/autotest_common.sh"
 source "$testdir/common.sh"
 
+function cleanup() {
+	killprocess $tgtpid
+	killprocess $smapid
+}
+
 function create_device() {
 	"$rootdir/scripts/sma-client.py" <<- EOF
 		{
@@ -17,7 +22,10 @@ function create_device() {
 	EOF
 }
 
-trap 'killprocess $smapid; exit 1' SIGINT SIGTERM EXIT
+trap 'cleanup; exit 1' SIGINT SIGTERM EXIT
+
+$rootdir/build/bin/spdk_tgt &
+tgtpid=$!
 
 # First check a single plugin with both its devices enabled in the config
 PYTHONPATH=$testdir/plugins $rootdir/scripts/sma.py -c <(
@@ -144,6 +152,5 @@ sma_waitforlisten
 [[ $(create_device nvme | jq -r '.handle') == 'nvme:plugin1-device1' ]]
 [[ $(create_device nvmf_tcp | jq -r '.handle') == 'nvmf_tcp:plugin2-device2' ]]
 
-killprocess $smapid
+cleanup
-
 trap - SIGINT SIGTERM EXIT