ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scripts/nvmf_perf: update ADQ configuration steps

Browse Source 
Initial ADQ configuration in script was created based
on "Nov 2020 ADQ with SPDK Configuration Guide".
Update using latest June 2022 guide version.

- Channel inline flow director configured using
  "devlink" for each TC separately instead of
  using ethtool setting
- Disable channel packet inspect optimize priv flag
- Enable busy poll private flags
- Run NIC IRQ affinity script again after creating
  TC's
- net.core.busy_poll and net.core.busy_read now both
  set to 1
- Set acceptor_poll_rate when creating transport layer

Change-Id: I05b2d0aa2ddd5fc249889fb5e5d6118963367b36
Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15584
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Jaroslaw Chachulski <jaroslawx.chachulski@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Karol Latecki 2022-11-23 08:22:39 +01:00 committed by Tomasz Zawadzki
parent a7d79228d7
commit 2b2dc8b32d
1 changed files with 33 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
scripts/perf/nvmf/run_nvmf.py
View File

@ -171,6 +171,10 @@ class Server:
self.log.info(" ".join(tc_qdisc_ingress_cmd)) self.log.info(" ".join(tc_qdisc_ingress_cmd))
self.exec_cmd(tc_qdisc_ingress_cmd) self.exec_cmd(tc_qdisc_ingress_cmd)
nic_bdf = os.path.basename(self.exec_cmd(["readlink", "-f", "/sys/class/net/%s/device" % nic_name]).strip())
self.exec_cmd(["sudo", "devlink", "dev", "param", "set", "pci/%s" % nic_bdf,
"name", "tc1_inline_fd", "value", "true", "cmode", "runtime"])
tc_filter_cmd = ["sudo", "tc", "filter", "add", "dev", nic_name, tc_filter_cmd = ["sudo", "tc", "filter", "add", "dev", nic_name,
"protocol", "ip", "ingress", "prio", "1", "flower", "protocol", "ip", "ingress", "prio", "1", "flower",
"dst_ip", "%s/32" % nic_ip, "ip_proto", "tcp", port_param, port, "dst_ip", "%s/32" % nic_ip, "ip_proto", "tcp", port_param, port,
@ -214,14 +218,14 @@ class Server:
try: try:
self.exec_cmd(["sudo", "ethtool", "-K", nic, self.exec_cmd(["sudo", "ethtool", "-K", nic,
"hw-tc-offload", "on"]) # Enable hardware TC offload "hw-tc-offload", "on"]) # Enable hardware TC offload
self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic, nic_bdf = self.exec_cmd(["bash", "-c", "source /sys/class/net/%s/device/uevent; echo $PCI_SLOT_NAME" % nic])
"channel-inline-flow-director", "on"]) # Enable Intel Flow Director
self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic, "fw-lldp-agent", "off"]) # Disable LLDP self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic, "fw-lldp-agent", "off"]) # Disable LLDP
# As temporary workaround for ADQ, channel packet inspection optimization is turned on during connection establishment. self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic, "channel-pkt-inspect-optimize", "off"])
# Then turned off before fio ramp_up expires in ethtool_after_fio_ramp(). # Following are suggested in ADQ Configuration Guide to be enabled for large block sizes,
self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic, # but can be used with 4k block sizes as well
"channel-pkt-inspect-optimize", "on"]) self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic, "channel-pkt-clean-bp-stop", "on"])
except CalledProcessError as e: self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic, "channel-pkt-clean-bp-stop-cfg", "on"])
except subprocess.CalledProcessError as e:
self.log.error("ERROR: failed to configure NIC port using ethtool!") self.log.error("ERROR: failed to configure NIC port using ethtool!")
self.log.error("%s resulted in error: %s" % (e.cmd, e.output)) self.log.error("%s resulted in error: %s" % (e.cmd, e.output))
self.log.info("Please update your NIC driver and firmware versions and try again.") self.log.info("Please update your NIC driver and firmware versions and try again.")
@ -259,9 +263,15 @@ class Server:
def configure_sysctl(self): def configure_sysctl(self):
self.log.info("Tuning sysctl settings...") self.log.info("Tuning sysctl settings...")
busy_read = 0
busy_poll = 0
if self.enable_adq and self.mode == "spdk":
busy_read = 1
busy_poll = 1
sysctl_opts = { sysctl_opts = {
"net.core.busy_poll": 0, "net.core.busy_poll": busy_poll,
"net.core.busy_read": 0, "net.core.busy_read": busy_read,
"net.core.somaxconn": 4096, "net.core.somaxconn": 4096,
"net.core.netdev_max_backlog": 8192, "net.core.netdev_max_backlog": 8192,
"net.ipv4.tcp_max_syn_backlog": 16384, "net.ipv4.tcp_max_syn_backlog": 16384,
@ -490,6 +500,7 @@ class Target(Server):
self.configure_system() self.configure_system()
if self.enable_adq: if self.enable_adq:
self.configure_adq() self.configure_adq()
self.configure_irq_affinity()
self.sys_config() self.sys_config()
def set_local_nic_info_helper(self): def set_local_nic_info_helper(self):
@ -578,14 +589,6 @@ class Target(Server):
"-d", "%s" % results_dir, "-l", "-p", "%s" % prefix, "-d", "%s" % results_dir, "-l", "-p", "%s" % prefix,
"-x", "-c", "%s" % run_time, "-t", "%s" % 1, "-r"]) "-x", "-c", "%s" % run_time, "-t", "%s" % 1, "-r"])
def ethtool_after_fio_ramp(self, fio_ramp_time):
time.sleep(fio_ramp_time//2)
nic_names = [self.get_nic_name_by_ip(n) for n in self.nic_ips]
for nic in nic_names:
self.log.info(nic)
self.exec_cmd(["sudo", "ethtool", "--set-priv-flags", nic,
"channel-pkt-inspect-optimize", "off"]) # Disable channel packet inspection optimization
def measure_pcm_memory(self, results_dir, pcm_file_name, ramp_time, run_time): def measure_pcm_memory(self, results_dir, pcm_file_name, ramp_time, run_time):
time.sleep(ramp_time) time.sleep(ramp_time)
cmd = ["pcm-memory", "1", "-csv=%s/%s" % (results_dir, pcm_file_name)] cmd = ["pcm-memory", "1", "-csv=%s/%s" % (results_dir, pcm_file_name)]
@ -1149,11 +1152,19 @@ class SPDKTarget(Target):
self.adq_configure_tc() self.adq_configure_tc()
# Create transport layer # Create transport layer
rpc.nvmf.nvmf_create_transport(self.client, trtype=self.transport, nvmf_transport_params = {
num_shared_buffers=self.num_shared_buffers, "client": self.client,
max_queue_depth=self.max_queue_depth, "trtype": self.transport,
dif_insert_or_strip=self.dif_insert_strip, "num_shared_buffers": self.num_shared_buffers,
sock_priority=self.adq_priority) "max_queue_depth": self.max_queue_depth,
"dif_insert_or_strip": self.dif_insert_strip,
"sock_priority": self.adq_priority
}
if self.enable_adq:
nvmf_transport_params["acceptor_poll_rate"] = 10000
rpc.nvmf.nvmf_create_transport(**nvmf_transport_params)
self.log.info("SPDK NVMeOF transport layer:") self.log.info("SPDK NVMeOF transport layer:")
rpc_client.print_dict(rpc.nvmf.nvmf_get_transports(self.client)) rpc_client.print_dict(rpc.nvmf.nvmf_get_transports(self.client))
@ -1724,10 +1735,6 @@ if __name__ == "__main__":
fio_ramp_time, fio_run_time)) fio_ramp_time, fio_run_time))
threads.append(power_daemon) threads.append(power_daemon)
if target_obj.enable_adq:
ethtool_thread = threading.Thread(target=target_obj.ethtool_after_fio_ramp, args=(fio_ramp_time,))
threads.append(ethtool_thread)
for t in threads: for t in threads:
t.start() t.start()
for t in threads: for t in threads: