env: add spdk_pci_device_allow() API

The env layer has a pci_allowed list, which specifies
that only a subset of PCI devices may be attached
by the associated process.

But that doesn't cover PCI devices that are hot-inserted
after the application starts, which is common for
storage/NVMe.

So add a new spdk_pci_device_allow() API which allows
an application to add new devices to the allowed list.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I7bd5ff428d84480d46bc236698daadd019b20b8e

Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6183
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: <dongx.yi@intel.com>

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## v21.04: (Upcoming Release)
 
+### env
+
+Added spdk_pci_device_allow API to allow applications to add PCI addresses to
+the allowed list after the application has started.
+
 ### nvme
 
 Added `spdk_nvme_qpair_get_optimal_poll_group` function and `qpair_get_optimal_poll_group`

include/spdk/env.h

@@ -1008,6 +1008,22 @@ void spdk_pci_device_detach(struct spdk_pci_device *device);
 int spdk_pci_device_attach(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb,
 			   void *enum_ctx, struct spdk_pci_addr *pci_address);
 
+/**
+ * Allow the specified PCI device to be probed by the calling process.
+ *
+ * When using spdk_pci_enumerate(), only devices with allowed PCI addresses will
+ * be probed.  By default, this is all PCI addresses, but the pci_allowed
+ * and pci_blocked environment options can override this behavior.
+ * This API enables the caller to allow a new PCI address that may have previously
+ * been blocked.
+ *
+ * \param pci_addr PCI address to allow
+ * \return 0 if successful
+ * \return -ENOMEM if environment-specific data structures cannot be allocated
+ * \return -EINVAL if specified PCI address is not valid
+ */
+int spdk_pci_device_allow(struct spdk_pci_addr *pci_addr);
+
 /**
  * Read \c len bytes from the PCI configuration space.
  *

lib/env_dpdk/pci.c

@@ -1072,3 +1072,37 @@ spdk_pci_device_get_type(const struct spdk_pci_device *dev)
 {
 	return dev->type;
 }
+
+int
+spdk_pci_device_allow(struct spdk_pci_addr *pci_addr)
+{
+	struct rte_devargs *da;
+	char devargs_str[128];
+
+	da = calloc(1, sizeof(*da));
+	if (da == NULL) {
+		SPDK_ERRLOG("could not allocate rte_devargs\n");
+		return -ENOMEM;
+	}
+
+	snprintf(devargs_str, sizeof(devargs_str), "pci:%04x:%02x:%02x:%x",
+		 pci_addr->domain, pci_addr->bus, pci_addr->dev, pci_addr->func);
+	if (rte_devargs_parse(da, devargs_str) != 0) {
+		SPDK_ERRLOG("rte_devargs_parse() failed on '%s'\n", devargs_str);
+		free(da);
+		return -EINVAL;
+	}
+	da->policy = RTE_DEV_ALLOWED;
+	/* Note: if a devargs already exists for this device address, it just gets
+	 * overridden.  So we do not need to check if the devargs already exists.
+	 * DPDK will take care of memory management for the devargs structure after
+	 * it has been inserted, so there's nothing SPDK needs to track.
+	 */
+	if (rte_devargs_insert(&da) != 0) {
+		SPDK_ERRLOG("rte_devargs_insert() failed on '%s'\n", devargs_str);
+		free(da);
+		return -EINVAL;
+	}
+
+	return 0;
+}

lib/env_dpdk/spdk_env_dpdk.map

@@ -79,6 +79,7 @@
 	spdk_pci_device_unclaim;
 	spdk_pci_device_detach;
 	spdk_pci_device_attach;
+	spdk_pci_device_allow;
 	spdk_pci_device_cfg_read;
 	spdk_pci_device_cfg_write;
 	spdk_pci_device_cfg_read8;