From e05299f1b2fd70730db107da11678a0065d2ab91 Mon Sep 17 00:00:00 2001 From: Karol Latecki Date: Wed, 18 May 2022 11:58:23 +0200 Subject: [PATCH] scripts/nvmf_perf: spread out subsystems evenly Spread out used block devices evenly between used initiators and then between initiators NIC interfaces, so that in result each initiator connects to the same number of subsystems. Previous version had problems spreading block devices / subsystems between interfaces in case division remainder was not 0. Signed-off-by: Karol Latecki Change-Id: I045dc9313b0613685fc997f821fe66238026a645 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12732 Reviewed-by: Konrad Sztyber Reviewed-by: Shuhei Matsumoto Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI --- scripts/perf/nvmf/run_nvmf.py | 161 +++++++++++++++++----------------- 1 file changed, 82 insertions(+), 79 deletions(-) diff --git a/scripts/perf/nvmf/run_nvmf.py b/scripts/perf/nvmf/run_nvmf.py index 6d22bd55b..02d1c63f5 100755 --- a/scripts/perf/nvmf/run_nvmf.py +++ b/scripts/perf/nvmf/run_nvmf.py @@ -420,6 +420,30 @@ class Target(Server): fh.close() self.log_print("Done zipping") + @staticmethod + def _chunks(input_list, chunks_no): + div, rem = divmod(len(input_list), chunks_no) + for i in range(chunks_no): + si = (div + 1) * (i if i < rem else rem) + div * (0 if i < rem else i - rem) + yield input_list[si:si + (div + 1 if i < rem else div)] + + def spread_bdevs(self, req_disks): + # Spread available block devices indexes: + # - evenly across available initiator systems + # - evenly across available NIC interfaces for + # each initiator + # Not NUMA aware. + ip_bdev_map = [] + initiator_chunks = self._chunks(range(0, req_disks), len(self.initiator_info)) + + for i, (init, init_chunk) in enumerate(zip(self.initiator_info, initiator_chunks)): + self.initiator_info[i]["bdev_range"] = init_chunk + init_chunks_list = list(self._chunks(init_chunk, len(init["target_nic_ips"]))) + for ip, nic_chunk in zip(self.initiator_info[i]["target_nic_ips"], init_chunks_list): + for c in nic_chunk: + ip_bdev_map.append((ip, c)) + return ip_bdev_map + @staticmethod def read_json_stats(file): with open(file, "r") as json_data: @@ -972,7 +996,7 @@ class KernelTarget(Target): def stop(self): nvmet_command(self.nvmet_bin, "clear") - def kernel_tgt_gen_subsystem_conf(self, nvme_list, address_list): + def kernel_tgt_gen_subsystem_conf(self, nvme_list): nvmet_cfg = { "ports": [], @@ -980,53 +1004,46 @@ class KernelTarget(Target): "subsystems": [], } - # Split disks between NIC IP's - disks_per_ip = int(len(nvme_list) / len(address_list)) - disk_chunks = [nvme_list[i * disks_per_ip:disks_per_ip + disks_per_ip * i] for i in range(0, len(address_list))] + for ip, bdev_num in self.spread_bdevs(len(nvme_list)): + port = str(4420 + bdev_num) + nqn = "nqn.2018-09.io.spdk:cnode%s" % bdev_num + serial = "SPDK00%s" % bdev_num + bdev_name = nvme_list[bdev_num] - # Add remaining drives - for i, disk in enumerate(nvme_list[disks_per_ip * len(address_list):]): - disk_chunks[i].append(disk) + nvmet_cfg["subsystems"].append({ + "allowed_hosts": [], + "attr": { + "allow_any_host": "1", + "serial": serial, + "version": "1.3" + }, + "namespaces": [ + { + "device": { + "path": bdev_name, + "uuid": "%s" % uuid.uuid4() + }, + "enable": 1, + "nsid": port + } + ], + "nqn": nqn + }) - subsys_no = 1 - port_no = 0 - for ip, chunk in zip(address_list, disk_chunks): - for disk in chunk: - nqn = "nqn.2018-09.io.spdk:cnode%s" % subsys_no - nvmet_cfg["subsystems"].append({ - "allowed_hosts": [], - "attr": { - "allow_any_host": "1", - "serial": "SPDK00%s" % subsys_no, - "version": "1.3" - }, - "namespaces": [ - { - "device": { - "path": disk, - "uuid": "%s" % uuid.uuid4() - }, - "enable": 1, - "nsid": subsys_no - } - ], - "nqn": nqn - }) + nvmet_cfg["ports"].append({ + "addr": { + "adrfam": "ipv4", + "traddr": ip, + "trsvcid": port, + "trtype": self.transport + }, + "portid": bdev_num, + "referrals": [], + "subsystems": [nqn] + }) - nvmet_cfg["ports"].append({ - "addr": { - "adrfam": "ipv4", - "traddr": ip, - "trsvcid": "%s" % (4420 + port_no), - "trtype": "%s" % self.transport - }, - "portid": subsys_no, - "referrals": [], - "subsystems": [nqn] - }) - subsys_no += 1 - port_no += 1 - self.subsystem_info_list.append([port_no, nqn, ip]) + self.subsystem_info_list.append([port, nqn, ip]) + self.subsys_no = len(self.subsystem_info_list) with open("kernel.conf", "w") as fh: fh.write(json.dumps(nvmet_cfg, indent=2)) @@ -1036,14 +1053,13 @@ class KernelTarget(Target): if self.null_block: print("Configuring with null block device.") - null_blk_list = ["/dev/nullb{}".format(x) for x in range(self.null_block)] - self.kernel_tgt_gen_subsystem_conf(null_blk_list, self.nic_ips) - self.subsys_no = len(null_blk_list) + nvme_list = ["/dev/nullb{}".format(x) for x in range(self.null_block)] else: print("Configuring with NVMe drives.") nvme_list = get_nvme_devices() - self.kernel_tgt_gen_subsystem_conf(nvme_list, self.nic_ips) - self.subsys_no = len(nvme_list) + + self.kernel_tgt_gen_subsystem_conf(nvme_list) + self.subsys_no = len(nvme_list) nvmet_command(self.nvmet_bin, "clear") nvmet_command(self.nvmet_bin, "restore kernel.conf") @@ -1162,40 +1178,27 @@ class SPDKTarget(Target): def spdk_tgt_add_subsystem_conf(self, ips=None, req_num_disks=None): self.log_print("Adding subsystems to config") - port = "4420" if not req_num_disks: req_num_disks = get_nvme_devices_count() - # Distribute bdevs between provided NICs - num_disks = range(0, req_num_disks) - if len(num_disks) == 1: - disks_per_ip = 1 - else: - disks_per_ip = int(len(num_disks) / len(ips)) - disk_chunks = [[*num_disks[i * disks_per_ip:disks_per_ip + disks_per_ip * i]] for i in range(0, len(ips))] + for ip, bdev_num in self.spread_bdevs(req_num_disks): + port = str(4420 + bdev_num) + nqn = "nqn.2018-09.io.spdk:cnode%s" % bdev_num + serial = "SPDK00%s" % bdev_num + bdev_name = "Nvme%sn1" % bdev_num - # Add remaining drives - for i, disk in enumerate(num_disks[disks_per_ip * len(ips):]): - disk_chunks[i].append(disk) + rpc.nvmf.nvmf_create_subsystem(self.client, nqn, serial, + allow_any_host=True, max_namespaces=8) + rpc.nvmf.nvmf_subsystem_add_ns(self.client, nqn, bdev_name) + rpc.nvmf.nvmf_subsystem_add_listener(self.client, + nqn=nqn, + trtype=self.transport, + traddr=ip, + trsvcid=port, + adrfam="ipv4") + self.subsystem_info_list.append([port, nqn, ip]) + self.subsys_no = len(self.subsystem_info_list) - # Create subsystems, add bdevs to namespaces, add listeners - for ip, chunk in zip(ips, disk_chunks): - for c in chunk: - nqn = "nqn.2018-09.io.spdk:cnode%s" % c - serial = "SPDK00%s" % c - bdev_name = "Nvme%sn1" % c - rpc.nvmf.nvmf_create_subsystem(self.client, nqn, serial, - allow_any_host=True, max_namespaces=8) - rpc.nvmf.nvmf_subsystem_add_ns(self.client, nqn, bdev_name) - - rpc.nvmf.nvmf_subsystem_add_listener(self.client, - nqn=nqn, - trtype=self.transport, - traddr=ip, - trsvcid=port, - adrfam="ipv4") - - self.subsystem_info_list.append([port, nqn, ip]) self.log_print("SPDK NVMeOF subsystem configuration:") rpc_client.print_dict(rpc.nvmf.nvmf_get_subsystems(self.client))