From 2c8baa46850003b624e2fb80f14a0a9c1bacc462 Mon Sep 17 00:00:00 2001 From: wawryk Date: Tue, 1 Jun 2021 14:24:31 +0200 Subject: [PATCH] scripts/nvmf: add io_uring engine option for Kernel Initiator Enable "kernel_engine" option for Kernel Initiator class to enable ability to use io_uring engine on initiator side. For NVMe-oF this has limitations as apparently it's not possible to enable polling on the initiator side, which in turn makes fio "hipri" option also not possible to use. Adding #TODO sections with notes in case this is later fixed in Kernel NVMe driver. Signed-off-by: wawryk Signed-off-by: Karol Latecki Change-Id: I31da52946692015237263abc77d8425b5eae5b98 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8134 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Reviewed-by: Ben Walker Reviewed-by: Ziye Yang Reviewed-by: Tomasz Zawadzki --- scripts/perf/nvmf/README.md | 8 +++++- scripts/perf/nvmf/run_nvmf.py | 48 ++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/scripts/perf/nvmf/README.md b/scripts/perf/nvmf/README.md index 8d61df4f8..0590748f7 100644 --- a/scripts/perf/nvmf/README.md +++ b/scripts/perf/nvmf/README.md @@ -167,7 +167,8 @@ There can be one or more `initiatorX` setting sections, depending on the test se "cpus_allowed_policy": "shared", "num_cores": 4, "cpu_frequency": 2100000, - "adq_enable": false + "adq_enable": false, + "kernel_engine": "io_uring" } ``` @@ -212,6 +213,11 @@ Optional, common: - irq_scripts_dir - path to scripts directory of Mellanox mlnx-tools package; Used to run set_irq_affinity.sh script. Default: /usr/src/local/mlnx-tools/ofed_scripts +- kernel_engine - Select fio ioengine mode to run tests. io_uring libraries and + io_uring capable fio binaries must be present on Initiator systems! + Available options: + - libaio (default) + - io_uring Optional, SPDK Initiator only: diff --git a/scripts/perf/nvmf/run_nvmf.py b/scripts/perf/nvmf/run_nvmf.py index 1820d8a82..608d019cb 100755 --- a/scripts/perf/nvmf/run_nvmf.py +++ b/scripts/perf/nvmf/run_nvmf.py @@ -802,7 +802,7 @@ rate_iops={rate_iops} ioengine = "%s/build/fio/spdk_bdev" % self.spdk_dir spdk_conf = "spdk_json_conf=%s/bdev.conf" % self.spdk_dir else: - ioengine = "libaio" + ioengine = self.ioengine spdk_conf = "" out = self.exec_cmd(["sudo", "nvme", "list", "|", "grep", "-E", "'SPDK|Linux'", "|", "awk", "'{print $1}'"]) @@ -837,6 +837,16 @@ rate_iops={rate_iops} fio_config = fio_conf_template.format(ioengine=ioengine, spdk_conf=spdk_conf, rw=rw, rwmixread=rwmixread, block_size=block_size, ramp_time=ramp_time, run_time=run_time, rate_iops=rate_iops) + + # TODO: hipri disabled for now, as it causes fio errors: + # io_u error on file /dev/nvme2n1: Operation not supported + # See comment in KernelInitiator class, kernel_init_connect() function + if hasattr(self, "ioengine") and "io_uring" in self.ioengine: + fio_config = fio_config + """ +fixedbufs=1 +registerfiles=1 +#hipri=1 +""" if num_jobs: fio_config = fio_config + "numjobs=%s \n" % num_jobs if self.cpus_allowed is not None: @@ -1172,13 +1182,25 @@ class KernelInitiator(Initiator): # Defaults self.extra_params = "" + self.ioengine = "libaio" if "extra_params" in initiator_config: self.extra_params = initiator_config["extra_params"] + if "kernel_engine" in initiator_config: + self.ioengine = initiator_config["kernel_engine"] + if "io_uring" in self.ioengine: + self.extra_params = "--nr-poll-queues=8" + def __del__(self): self.ssh_connection.close() + def get_connected_nvme_list(self): + json_obj = json.loads(self.exec_cmd(["sudo", "nvme", "list", "-o", "json"])) + nvme_list = [os.path.basename(x["DevicePath"]) for x in json_obj["Devices"] + if "SPDK" in x["ModelNumber"] or "Linux" in x["ModelNumber"]] + return nvme_list + def kernel_init_connect(self): self.log_print("Below connection attempts may result in error messages, this is expected!") for subsystem in self.subsystem_info_list: @@ -1187,6 +1209,30 @@ class KernelInitiator(Initiator): "-s", subsystem[0], "-n", subsystem[1], "-a", subsystem[2], self.extra_params]) time.sleep(2) + if "io_uring" in self.ioengine: + self.log_print("Setting block layer settings for io_uring.") + + # TODO: io_poll=1 and io_poll_delay=-1 params not set here, because + # apparently it's not possible for connected subsystems. + # Results in "error: Invalid argument" + block_sysfs_settings = { + "iostats": "0", + "rq_affinity": "0", + "nomerges": "2" + } + + for disk in self.get_connected_nvme_list(): + sysfs = os.path.join("/sys/block", disk, "queue") + for k, v in block_sysfs_settings.items(): + sysfs_opt_path = os.path.join(sysfs, k) + try: + self.exec_cmd(["sudo", "bash", "-c", "echo %s > %s" % (v, sysfs_opt_path)], stderr_redirect=True) + except subprocess.CalledProcessError as e: + self.log_print("Warning: command %s failed due to error %s. %s was not set!" % (e.cmd, e.output, v)) + finally: + _ = self.exec_cmd(["sudo", "cat", "%s" % (sysfs_opt_path)]) + self.log_print("%s=%s" % (sysfs_opt_path, _)) + def kernel_init_disconnect(self): for subsystem in self.subsystem_info_list: self.exec_cmd(["sudo", self.nvmecli_bin, "disconnect", "-n", subsystem[1]])