scripts: move python modules to python directory

Up until now, importing an SPDK RPC python module was just a matter of
`import rpc`.  It's fine until there's another module called `rpc`
installed on the system, in which case it's impossible to import both of
them.  Therefore, to avoid this problem, all of the modules were moved
to a separate directory under the "spdk" namespace.

The decision to move to a location under a separate directory was
motivated by the fact that a directory called scripts/spdk would look
pretty confusing.  Moreover, it should make it also easier to package
these scripts as a python package.

Other than moving the packages, all of the imports were updated to
reflect these changes.  Files under python now use relative imports,
while those under scripts/ use the "spdk" namespace and have their
PYTHONPATH extended with python directory.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Ib43dee73921d590a551dd83885e22870e72451cf
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9692
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

doc/jsonrpc.md

@@ -139,7 +139,7 @@ argument that adds new RPC calls (subparsers.add_parser()).  The new parsers sho
 functions registered within the external module using the SPDK_RPC_REGISTER() macro.  Example:
 
 ~~~python
-from rpc.client import print_json
+from spdk.rpc.client import print_json
 
 
 def example_create(client, num_blocks, block_size, name=None, uuid=None):

examples/interrupt_tgt/interrupt_plugin.py

@@ -1,4 +1,4 @@
-from rpc.client import print_json
+from spdk.rpc.client import print_json
 
 
 def reactor_set_interrupt_mode(args):

python/spdk/spdkcli/ui_node.py

@@ -1,6 +1,6 @@
 from configshell_fb import ConfigNode, ExecutionError
 from uuid import UUID
-from rpc.client import JSONRPCException
+from ..rpc.client import JSONRPCException
 import json
 
 

python/spdk/spdkcli/ui_node_iscsi.py

@@ -1,5 +1,5 @@
 from configshell_fb import ExecutionError
-from rpc.client import JSONRPCException
+from ..rpc.client import JSONRPCException
 from .ui_node import UINode
 
 

python/spdk/spdkcli/ui_node_nvmf.py

@@ -1,4 +1,4 @@
-from rpc.client import JSONRPCException
+from ..rpc.client import JSONRPCException
 from .ui_node import UINode
 
 

python/spdk/spdkcli/ui_root.py

@@ -1,8 +1,7 @@
 from .ui_node import UINode, UIBdevs, UILvolStores, UIVhosts
 from .ui_node_nvmf import UINVMf
 from .ui_node_iscsi import UIISCSI
-import rpc.client
-import rpc
+from .. import rpc
 from functools import wraps
 
 

rpmbuild/spdk.spec

@@ -33,6 +33,8 @@ Requires: zlib
 Requires: %(echo "%{requirements_list}")
 %endif
 
+BuildRequires: python3-devel
+
 %if %{build_requirements}
 BuildRequires: %(echo "%{build_requirements_list}")
 %endif
@@ -82,6 +84,7 @@ mkdir -p %{buildroot}/usr/libexec/spdk
 mkdir -p %{buildroot}/etc/bash_completion.d
 mkdir -p %{buildroot}/etc/profile.d
 mkdir -p %{buildroot}/etc/ld.so.conf.d
+mkdir -p %{buildroot}%{python3_sitelib}
 
 cat <<-EOF > %{buildroot}/etc/ld.so.conf.d/spdk.conf
 /usr/local/lib
@@ -96,6 +99,7 @@ export PATH
 EOF
 
 cp -a scripts %{buildroot}/usr/libexec/spdk/scripts
+cp -a python/spdk %{buildroot}%{python3_sitelib}
 ln -s /usr/libexec/spdk/scripts/bash-completion/spdk %{buildroot}/etc/bash_completion.d/
 
 # We need to take into the account the fact that most of the scripts depend on being
@@ -111,6 +115,7 @@ ln -s /usr/local/include %{buildroot}/usr/libexec/spdk
 /etc/bash_completion.d/*
 /usr/libexec/spdk/*
 /usr/local/bin/*
+%{python3_sitelib}/spdk/*
 
 
 %package devel

scripts/iostat.py

@@ -1,10 +1,14 @@
 #!/usr/bin/env python3
 
 import logging
+import os
 import sys
 import argparse
 import time
-import rpc
+
+sys.path.append(os.path.dirname(__file__) + '/../python')
+
+import spdk.rpc as rpc  # noqa
 
 
 SPDK_CPU_STAT = "/proc/stat"

scripts/perf/nvmf/run_nvmf.py

@@ -17,11 +17,13 @@ from collections import OrderedDict
 
 import paramiko
 import pandas as pd
-
-import rpc
-import rpc.client
 from common import *
 
+sys.path.append(os.path.dirname(__file__) + '/../../../python')
+
+import spdk.rpc as rpc  # noqa
+import spdk.rpc.client as rpc_client  # noqa
+
 
 class Server:
     def __init__(self, name, general_config, server_config):
@@ -1101,7 +1103,7 @@ class SPDKTarget(Target):
                                        dif_insert_or_strip=self.dif_insert_strip,
                                        sock_priority=self.adq_priority)
         self.log_print("SPDK NVMeOF transport layer:")
-        rpc.client.print_dict(rpc.nvmf.nvmf_get_transports(self.client))
+        rpc_client.print_dict(rpc.nvmf.nvmf_get_transports(self.client))
 
         if self.null_block:
             self.spdk_tgt_add_nullblock(self.null_block)
@@ -1124,7 +1126,7 @@ class SPDKTarget(Target):
             rpc.bdev.bdev_null_create(self.client, 102400, block_size + md_size, "Nvme{}n1".format(i),
                                       dif_type=self.null_block_dif_type, md_size=md_size)
         self.log_print("SPDK Bdevs configuration:")
-        rpc.client.print_dict(rpc.bdev.bdev_get_bdevs(self.client))
+        rpc_client.print_dict(rpc.bdev.bdev_get_bdevs(self.client))
 
     def spdk_tgt_add_nvme_conf(self, req_num_disks=None):
         self.log_print("Adding NVMe bdevs to config via RPC")
@@ -1143,7 +1145,7 @@ class SPDKTarget(Target):
             rpc.bdev.bdev_nvme_attach_controller(self.client, name="Nvme%s" % i, trtype="PCIe", traddr=bdf)
 
         self.log_print("SPDK Bdevs configuration:")
-        rpc.client.print_dict(rpc.bdev.bdev_get_bdevs(self.client))
+        rpc_client.print_dict(rpc.bdev.bdev_get_bdevs(self.client))
 
     def spdk_tgt_add_subsystem_conf(self, ips=None, req_num_disks=None):
         self.log_print("Adding subsystems to config")
@@ -1182,7 +1184,7 @@ class SPDKTarget(Target):
 
                 self.subsystem_info_list.append([port, nqn, ip])
         self.log_print("SPDK NVMeOF subsystem configuration:")
-        rpc.client.print_dict(rpc.nvmf.nvmf_get_subsystems(self.client))
+        rpc_client.print_dict(rpc.nvmf.nvmf_get_subsystems(self.client))
 
     def bpf_start(self):
         self.log_print("Starting BPF Trace scripts: %s" % self.bpf_scripts)
@@ -1216,13 +1218,13 @@ class SPDKTarget(Target):
             if os.path.exists("/var/tmp/spdk.sock"):
                 break
             time.sleep(1)
-        self.client = rpc.client.JSONRPCClient("/var/tmp/spdk.sock")
+        self.client = rpc_client.JSONRPCClient("/var/tmp/spdk.sock")
 
         if self.enable_zcopy:
             rpc.sock.sock_impl_set_options(self.client, impl_name="posix",
                                            enable_zerocopy_send_server=True)
             self.log_print("Target socket options:")
-            rpc.client.print_dict(rpc.sock.sock_impl_get_options(self.client, impl_name="posix"))
+            rpc_client.print_dict(rpc.sock.sock_impl_get_options(self.client, impl_name="posix"))
 
         if self.enable_adq:
             rpc.sock.sock_impl_set_options(self.client, impl_name="posix", enable_placement_id=1)

scripts/rpc.py

@@ -1,12 +1,9 @@
 #!/usr/bin/env python3
 
-from rpc.client import print_dict, print_json, JSONRPCException
-from rpc.helpers import deprecated_aliases
-
 import logging
 import argparse
 import importlib
-import rpc
+import os
 import sys
 import shlex
 import json
@@ -16,6 +13,12 @@ try:
 except ImportError:
     from pipes import quote
 
+sys.path.append(os.path.dirname(__file__) + '/../python')
+
+import spdk.rpc as rpc  # noqa
+from spdk.rpc.client import print_dict, print_json, JSONRPCException  # noqa
+from spdk.rpc.helpers import deprecated_aliases  # noqa
+
 
 def print_array(a):
     print(" ".join((quote(v) for v in a)))

scripts/rpc_http_proxy.py

@@ -4,6 +4,7 @@ import argparse
 import base64
 import errno
 import json
+import os
 import socket
 import ssl
 import sys
@@ -12,7 +13,10 @@ try:
 except ImportError:
     from http.server import HTTPServer
     from http.server import BaseHTTPRequestHandler
-from rpc.client import print_json
+
+sys.path.append(os.path.dirname(__file__) + '/../python')
+
+from spdk.rpc.client import print_json  # noqa
 
 rpc_sock = None
 

scripts/spdkcli.py

@@ -1,11 +1,15 @@
 #!/usr/bin/env python3
+import os
 import sys
 import argparse
 from configshell_fb import ConfigShell, shell, ExecutionError
 from pyparsing import (alphanums, Optional, Suppress, Word, Regex,
                        removeQuotes, dblQuotedString, OneOrMore)
-from rpc.client import JSONRPCException, JSONRPCClient
-from spdkcli import UIRoot
+
+sys.path.append(os.path.dirname(__file__) + '/../python')
+
+from spdk.rpc.client import JSONRPCException, JSONRPCClient  # noqa
+from spdk.spdkcli import UIRoot  # noqa
 
 
 def add_quotes_to_shell(spdk_shell):

test/app/fuzz/common/fuzz_rpc.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 
-from rpc.client import print_dict, JSONRPCException
+from spdk.rpc.client import print_dict, JSONRPCException
 
 import logging
 import argparse
-import rpc
+import spdk.rpc as rpc
 import sys
 import shlex
 

test/bdev/bdevio/tests.py

@@ -5,8 +5,8 @@ import sys
 import shlex
 
 try:
-    from rpc.client import print_dict, JSONRPCException
-    import rpc
+    from spdk.rpc.client import print_dict, JSONRPCException
+    import spdk.rpc as rpc
 except ImportError:
     print("SPDK RPC library missing. Please add spdk/scripts/ directory to PYTHONPATH:")
     print("'export PYTHONPATH=$PYTHONPATH:./spdk/scripts/'")

test/bdev/bdevperf/bdevperf.py

@@ -6,8 +6,8 @@ import sys
 import shlex
 
 try:
-    from rpc.client import print_dict, JSONRPCException
-    import rpc
+    from spdk.rpc.client import print_dict, JSONRPCException
+    import spdk.rpc as rpc
 except ImportError:
     print("SPDK RPC library missing. Please add spdk/scripts/ directory to PYTHONPATH:")
     print("'export PYTHONPATH=$PYTHONPATH:./spdk/scripts/'")

test/common/autotest_common.sh

@@ -171,7 +171,7 @@ export PCI_BLOCK_SYNC_ON_RESET=yes
 
 # Export PYTHONPATH with addition of RPC framework. New scripts can be created
 # specific use cases for tests.
-export PYTHONPATH=$PYTHONPATH:$rootdir/scripts
+export PYTHONPATH=$PYTHONPATH:$rootdir/python
 
 # Don't create Python .pyc files. When running with sudo these will be
 # created with root ownership and can cause problems when cleaning the repository.

test/event/scheduler/scheduler_plugin.py

@@ -1,4 +1,4 @@
-from rpc.client import print_json
+from spdk.rpc.client import print_json
 
 
 def thread_create(args):

test/interrupt/reactor_set_interrupt.sh

@@ -5,7 +5,7 @@ rootdir=$(readlink -f $testdir/../..)
 source $rootdir/test/common/autotest_common.sh
 source $testdir/interrupt_common.sh
 
-export PYTHONPATH=$rootdir/examples/interrupt_tgt
+export PYTHONPATH=$PYTHONPATH:$rootdir/examples/interrupt_tgt
 
 function reactor_set_intr_mode() {
 	local spdk_pid=$1

test/interrupt/reap_unregistered_poller.sh

@@ -5,7 +5,7 @@ rootdir=$(readlink -f $testdir/../..)
 source $rootdir/test/common/autotest_common.sh
 source $testdir/interrupt_common.sh
 
-export PYTHONPATH=$rootdir/examples/interrupt_tgt
+export PYTHONPATH=$PYTHONPATH:$rootdir/examples/interrupt_tgt
 
 # Set reactors with intr_tgt in intr mode
 start_intr_tgt

test/json_config/clear_config.py

@@ -4,9 +4,8 @@ import os
 import sys
 import argparse
 import logging
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../scripts"))
-import rpc   # noqa
-from rpc.client import print_dict, JSONRPCException  # noqa
+import spdk.rpc as rpc
+from spdk.rpc.client import print_dict, JSONRPCException
 
 
 def get_bdev_name_key(bdev):

test/nvmf/target/multitarget_rpc.py

@@ -2,11 +2,11 @@
 
 # Not for use in production. Please see the changelog for v19.10.
 
-from rpc.client import print_dict, JSONRPCException
+from spdk.rpc.client import print_dict, JSONRPCException
 
 import logging
 import argparse
-import rpc
+import spdk.rpc as rpc
 import sys
 import shlex
 

test/rpc/rpc.sh

@@ -42,7 +42,7 @@ spdk_pid=$!
 trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT
 waitforlisten $spdk_pid
 
-export PYTHONPATH=$testdir
+export PYTHONPATH=$PYTHONPATH:$testdir
 
 # basic integrity test
 rpc=rpc_cmd

test/rpc/rpc_plugin.py

@@ -1,4 +1,4 @@
-from rpc.client import print_json
+from spdk.rpc.client import print_json
 
 
 def malloc_create(args):