ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scripts/nvmf_perf: add roce_enable option to configuration file

Browse Source 
Parse the config file for "irdma_roce_enable" flag to load irdma
driver with ROCE network protocol enabled on SPDK Target and
SPDK Initiator machines.

Signed-off-by: Jaroslaw Chachulski <jaroslawx.chachulski@intel.com>
Change-Id: If81c0ec494c13a561c6c780105ee4caba053001a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15343
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Pawel Piatek <pawelx.piatek@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Jaroslaw Chachulski 2022-11-08 09:51:24 -05:00 committed by Konrad Sztyber
parent 9579995cb3
commit 1c7b4b7b8a
2 changed files with 38 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scripts/perf/nvmf/README.md
View File

@ -59,7 +59,8 @@ The following sub-chapters describe each configuration section in more detail.
"username": "user",
"password": "password",
"transport": "transport_type",
"skip_spdk_install": bool
"skip_spdk_install": bool,
"irdma_roce_enable": bool
}
```
@ -76,6 +77,9 @@ Optional:
is already in place on Initiator systems and there's no need to re-build it,
then set this option to true.
Default: false.
- irdma_roce_enable - loads irdma driver with RoCEv2 network protocol enabled on Target and
Initiator machines. This option applies only to system with Intel E810 NICs.
Default: false
### Target System Configuration

35
scripts/perf/nvmf/run_nvmf.py
View File

@ -37,6 +37,7 @@ class Server:
self.transport = general_config["transport"].lower()
self.nic_ips = server_config["nic_ips"]
self.mode = server_config["mode"]
self.irdma_roce_enable = False
self.log = logging.getLogger(self.name)
@ -63,6 +64,8 @@ class Server:
self.irq_settings.update(server_config["irq_settings"])
if "tuned_profile" in server_config:
self.tuned_profile = server_config["tuned_profile"]
if "irdma_roce_enable" in general_config:
self.irdma_roce_enable = general_config["irdma_roce_enable"]
if not re.match(r'^[A-Za-z0-9\-]+$', name):
self.log.info("Please use a name which contains only letters, numbers or dashes")
@ -126,11 +129,38 @@ class Server:
self.configure_cpu_governor()
self.configure_irq_affinity(**self.irq_settings)
RDMA_PROTOCOL_IWARP = 0
RDMA_PROTOCOL_ROCE = 1
RDMA_PROTOCOL_UNKNOWN = -1
def check_rdma_protocol(self):
try:
roce_ena = self.exec_cmd(["cat", "/sys/module/irdma/parameters/roce_ena"])
roce_ena = roce_ena.decode().strip()
if roce_ena == "0":
return self.RDMA_PROTOCOL_IWARP
else:
return self.RDMA_PROTOCOL_ROCE
except CalledProcessError as e:
self.log.error("ERROR: failed to check RDMA protocol!")
self.log.error("%s resulted in error: %s" % (e.cmd, e.output))
return self.RDMA_PROTOCOL_UNKNOWN
def load_drivers(self):
self.log.info("Loading drivers")
self.exec_cmd(["sudo", "modprobe", "-a",
"nvme-%s" % self.transport,
"nvmet-%s" % self.transport])
current_mode = self.check_rdma_protocol()
if current_mode == self.RDMA_PROTOCOL_UNKNOWN:
self.log.error("ERROR: failed to check RDMA protocol mode")
return
if self.irdma_roce_enable and current_mode == self.RDMA_PROTOCOL_IWARP:
self.reload_driver("irdma", "roce_ena=1")
elif self.irdma_roce_enable and current_mode == self.RDMA_PROTOCOL_ROCE:
self.log.info("Leaving irdma driver with RoCE enabled")
else:
self.reload_driver("irdma", "roce_ena=0")
def configure_adq(self):
self.adq_load_modules()
@ -200,10 +230,11 @@ class Server:
self.log.info(xps_cmd)
self.exec_cmd(xps_cmd)
def reload_driver(self, driver):
def reload_driver(self, driver, *modprobe_args):
try:
self.exec_cmd(["sudo", "rmmod", driver])
self.exec_cmd(["sudo", "modprobe", driver])
self.exec_cmd(["sudo", "modprobe", driver, *modprobe_args])
except CalledProcessError as e:
self.log.error("ERROR: failed to reload %s module!" % driver)
self.log.error("%s resulted in error: %s" % (e.cmd, e.output))