scripts/nvmf_perf: spread out subsystems evenly

Spread out used block devices evenly between
used initiators and then between initiators NIC
interfaces, so that in result each initiator
connects to the same number of subsystems.

Previous version had problems spreading block
devices / subsystems between interfaces in case
division remainder was not 0.

Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Change-Id: I045dc9313b0613685fc997f821fe66238026a645
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12732
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>

scripts/perf/nvmf/run_nvmf.py

@@ -420,6 +420,30 @@ class Target(Server):
         fh.close()
         self.log_print("Done zipping")
 
+    @staticmethod
+    def _chunks(input_list, chunks_no):
+        div, rem = divmod(len(input_list), chunks_no)
+        for i in range(chunks_no):
+            si = (div + 1) * (i if i < rem else rem) + div * (0 if i < rem else i - rem)
+            yield input_list[si:si + (div + 1 if i < rem else div)]
+
+    def spread_bdevs(self, req_disks):
+        # Spread available block devices indexes:
+        # - evenly across available initiator systems
+        # - evenly across available NIC interfaces for
+        #   each initiator
+        # Not NUMA aware.
+        ip_bdev_map = []
+        initiator_chunks = self._chunks(range(0, req_disks), len(self.initiator_info))
+
+        for i, (init, init_chunk) in enumerate(zip(self.initiator_info, initiator_chunks)):
+            self.initiator_info[i]["bdev_range"] = init_chunk
+            init_chunks_list = list(self._chunks(init_chunk, len(init["target_nic_ips"])))
+            for ip, nic_chunk in zip(self.initiator_info[i]["target_nic_ips"], init_chunks_list):
+                for c in nic_chunk:
+                    ip_bdev_map.append((ip, c))
+        return ip_bdev_map
+
     @staticmethod
     def read_json_stats(file):
         with open(file, "r") as json_data:
@@ -972,7 +996,7 @@ class KernelTarget(Target):
     def stop(self):
         nvmet_command(self.nvmet_bin, "clear")
 
-    def kernel_tgt_gen_subsystem_conf(self, nvme_list, address_list):
+    def kernel_tgt_gen_subsystem_conf(self, nvme_list):
 
         nvmet_cfg = {
             "ports": [],
@@ -980,53 +1004,46 @@ class KernelTarget(Target):
             "subsystems": [],
         }
 
-        # Split disks between NIC IP's
+        for ip, bdev_num in self.spread_bdevs(len(nvme_list)):
-        disks_per_ip = int(len(nvme_list) / len(address_list))
+            port = str(4420 + bdev_num)
-        disk_chunks = [nvme_list[i * disks_per_ip:disks_per_ip + disks_per_ip * i] for i in range(0, len(address_list))]
+            nqn = "nqn.2018-09.io.spdk:cnode%s" % bdev_num
+            serial = "SPDK00%s" % bdev_num
+            bdev_name = nvme_list[bdev_num]
 
-        # Add remaining drives
+            nvmet_cfg["subsystems"].append({
-        for i, disk in enumerate(nvme_list[disks_per_ip * len(address_list):]):
+                "allowed_hosts": [],
-            disk_chunks[i].append(disk)
+                "attr": {
+                    "allow_any_host": "1",
+                    "serial": serial,
+                    "version": "1.3"
+                },
+                "namespaces": [
+                    {
+                        "device": {
+                            "path": bdev_name,
+                            "uuid": "%s" % uuid.uuid4()
+                        },
+                        "enable": 1,
+                        "nsid": port
+                    }
+                ],
+                "nqn": nqn
+            })
 
-        subsys_no = 1
+            nvmet_cfg["ports"].append({
-        port_no = 0
+                "addr": {
-        for ip, chunk in zip(address_list, disk_chunks):
+                    "adrfam": "ipv4",
-            for disk in chunk:
+                    "traddr": ip,
-                nqn = "nqn.2018-09.io.spdk:cnode%s" % subsys_no
+                    "trsvcid": port,
-                nvmet_cfg["subsystems"].append({
+                    "trtype": self.transport
-                    "allowed_hosts": [],
+                },
-                    "attr": {
+                "portid": bdev_num,
-                        "allow_any_host": "1",
+                "referrals": [],
-                        "serial": "SPDK00%s" % subsys_no,
+                "subsystems": [nqn]
-                        "version": "1.3"
+            })
-                    },
-                    "namespaces": [
-                        {
-                            "device": {
-                                "path": disk,
-                                "uuid": "%s" % uuid.uuid4()
-                            },
-                            "enable": 1,
-                            "nsid": subsys_no
-                        }
-                    ],
-                    "nqn": nqn
-                })
 
-                nvmet_cfg["ports"].append({
+            self.subsystem_info_list.append([port, nqn, ip])
-                    "addr": {
+        self.subsys_no = len(self.subsystem_info_list)
-                        "adrfam": "ipv4",
-                        "traddr": ip,
-                        "trsvcid": "%s" % (4420 + port_no),
-                        "trtype": "%s" % self.transport
-                    },
-                    "portid": subsys_no,
-                    "referrals": [],
-                    "subsystems": [nqn]
-                })
-                subsys_no += 1
-                port_no += 1
-                self.subsystem_info_list.append([port_no, nqn, ip])
 
         with open("kernel.conf", "w") as fh:
             fh.write(json.dumps(nvmet_cfg, indent=2))
@@ -1036,14 +1053,13 @@ class KernelTarget(Target):
 
         if self.null_block:
             print("Configuring with null block device.")
-            null_blk_list = ["/dev/nullb{}".format(x) for x in range(self.null_block)]
+            nvme_list = ["/dev/nullb{}".format(x) for x in range(self.null_block)]
-            self.kernel_tgt_gen_subsystem_conf(null_blk_list, self.nic_ips)
-            self.subsys_no = len(null_blk_list)
         else:
             print("Configuring with NVMe drives.")
             nvme_list = get_nvme_devices()
-            self.kernel_tgt_gen_subsystem_conf(nvme_list, self.nic_ips)
+
-            self.subsys_no = len(nvme_list)
+        self.kernel_tgt_gen_subsystem_conf(nvme_list)
+        self.subsys_no = len(nvme_list)
 
         nvmet_command(self.nvmet_bin, "clear")
         nvmet_command(self.nvmet_bin, "restore kernel.conf")
@@ -1162,40 +1178,27 @@ class SPDKTarget(Target):
 
     def spdk_tgt_add_subsystem_conf(self, ips=None, req_num_disks=None):
         self.log_print("Adding subsystems to config")
-        port = "4420"
         if not req_num_disks:
             req_num_disks = get_nvme_devices_count()
 
-        # Distribute bdevs between provided NICs
+        for ip, bdev_num in self.spread_bdevs(req_num_disks):
-        num_disks = range(0, req_num_disks)
+            port = str(4420 + bdev_num)
-        if len(num_disks) == 1:
+            nqn = "nqn.2018-09.io.spdk:cnode%s" % bdev_num
-            disks_per_ip = 1
+            serial = "SPDK00%s" % bdev_num
-        else:
+            bdev_name = "Nvme%sn1" % bdev_num
-            disks_per_ip = int(len(num_disks) / len(ips))
-        disk_chunks = [[*num_disks[i * disks_per_ip:disks_per_ip + disks_per_ip * i]] for i in range(0, len(ips))]
 
-        # Add remaining drives
+            rpc.nvmf.nvmf_create_subsystem(self.client, nqn, serial,
-        for i, disk in enumerate(num_disks[disks_per_ip * len(ips):]):
+                                           allow_any_host=True, max_namespaces=8)
-            disk_chunks[i].append(disk)
+            rpc.nvmf.nvmf_subsystem_add_ns(self.client, nqn, bdev_name)
+            rpc.nvmf.nvmf_subsystem_add_listener(self.client,
+                                                 nqn=nqn,
+                                                 trtype=self.transport,
+                                                 traddr=ip,
+                                                 trsvcid=port,
+                                                 adrfam="ipv4")
+            self.subsystem_info_list.append([port, nqn, ip])
+        self.subsys_no = len(self.subsystem_info_list)
 
-        # Create subsystems, add bdevs to namespaces, add listeners
-        for ip, chunk in zip(ips, disk_chunks):
-            for c in chunk:
-                nqn = "nqn.2018-09.io.spdk:cnode%s" % c
-                serial = "SPDK00%s" % c
-                bdev_name = "Nvme%sn1" % c
-                rpc.nvmf.nvmf_create_subsystem(self.client, nqn, serial,
-                                               allow_any_host=True, max_namespaces=8)
-                rpc.nvmf.nvmf_subsystem_add_ns(self.client, nqn, bdev_name)
-
-                rpc.nvmf.nvmf_subsystem_add_listener(self.client,
-                                                     nqn=nqn,
-                                                     trtype=self.transport,
-                                                     traddr=ip,
-                                                     trsvcid=port,
-                                                     adrfam="ipv4")
-
-                self.subsystem_info_list.append([port, nqn, ip])
         self.log_print("SPDK NVMeOF subsystem configuration:")
         rpc_client.print_dict(rpc.nvmf.nvmf_get_subsystems(self.client))