scripts/rpc.py: add new parameter to retry connecting

This can potentially replace waitforlisten() in our test scripts.

$ ./scripts/rpc.py -h
  -r CONN_RETRIES       Retry connecting to the RPC server N times with
                        0.2s interval. Default: 0
[...]

$ ./scripts/rpc.py -r 10 spdk_get_version
<SNIP the usual python exception>
rpc.client.JSONRPCException: Error while connecting to
/var/tmp/spdk.sock
Error details: [Errno -2] Name or service not known

real    0m2.452s
user    0m0.242s
sys     0m0.020s

Change-Id: I204da366360647fcce8f8de0eb25b0f576aabc5b
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2751
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

scripts/rpc.py

@@ -32,6 +32,9 @@ if __name__ == "__main__":
     parser.add_argument('-t', dest='timeout',
                         help='Timeout as a floating point number expressed in seconds waiting for response. Default: 60.0',
                         default=60.0, type=float)
+    parser.add_argument('-r', dest='conn_retries',
+                        help='Retry connecting to the RPC server N times with 0.2s interval. Default: 0',
+                        default=0, type=int)
     parser.add_argument('-v', dest='verbose', action='store_const', const="INFO",
                         help='Set verbose mode to INFO', default="ERROR")
     parser.add_argument('--verbose', dest='verbose', choices=['DEBUG', 'INFO', 'ERROR'],
@@ -2444,7 +2447,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
             try:
                 tmp_args.client = rpc.client.JSONRPCClient(
                     tmp_args.server_addr, tmp_args.port, tmp_args.timeout,
-                    log_level=getattr(logging, tmp_args.verbose.upper()))
+                    log_level=getattr(logging, tmp_args.verbose.upper()), conn_retries=tmp_args.conn_retries)
                 call_rpc_func(tmp_args)
                 print("**STATUS=0", flush=True)
             except JSONRPCException as ex:
@@ -2457,7 +2460,9 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
         print_json = null_print
         print_array = null_print
     else:
-        args.client = rpc.client.JSONRPCClient(args.server_addr, args.port, args.timeout, log_level=getattr(logging, args.verbose.upper()))
+        args.client = rpc.client.JSONRPCClient(args.server_addr, args.port, args.timeout,
+                                               log_level=getattr(logging, args.verbose.upper()),
+                                               conn_retries=args.conn_retries)
     if hasattr(args, 'func'):
         try:
             call_rpc_func(args)

scripts/rpc/client.py

@@ -28,11 +28,31 @@ class JSONRPCClient(object):
         self._logger = logging.getLogger("JSONRPCClient(%s)" % addr)
         self._logger.addHandler(ch)
         self.log_set_level(kwargs.get('log_level', logging.ERROR))
+        connect_retries = kwargs.get('conn_retries', 0)
 
         self.timeout = timeout
         self._request_id = 0
         self._recv_buf = ""
         self._reqs = []
+
+        for i in range(connect_retries):
+            try:
+                self._connect(addr, port)
+                return
+            except Exception as e:
+                # ignore and retry in 200ms
+                time.sleep(0.2)
+
+        # try one last time without try/except
+        self._connect(addr, port)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.close()
+
+    def _connect(self, addr, port):
         try:
             if os.path.exists(addr):
                 self._logger.debug("Trying to connect to UNIX socket: %s", addr)
@@ -55,12 +75,6 @@ class JSONRPCClient(object):
             raise JSONRPCException("Error while connecting to %s\n"
                                    "Error details: %s" % (addr, ex))
 
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exception_type, exception_value, traceback):
-        self.close()
-
     def get_logger(self):
         return self._logger