nvme: add API to allocate CMB I/O buffers

Change-Id: I2a3c7a272dc08be5a5ecb4339622816482c4cbb0
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
Signed-off-by: Stephen Bates <sbates@raithlin.com>
Reviewed-on: https://review.gerrithub.io/397036
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>

CHANGELOG.md

@@ -7,6 +7,10 @@
 The Rpc configuration file section, which was deprecated in v18.01, has been removed.
 Users should switch to the `-r` command-line parameter instead.
 
+### NVMe Driver
+
+EXPERIMENTAL: Adds support for WDS and RDS capable CMBs in NVMe controllers. This support is
+experimental pending a functional allocator to free and reallocate CMB buffers.
 
 ## v18.01: Blobstore Thin Provisioning
 

include/spdk/nvme.h

@@ -916,6 +916,25 @@ int spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload
 				    int slot, enum spdk_nvme_fw_commit_action commit_action,
 				    struct spdk_nvme_status *completion_status);
 
+/**
+ * \brief Allocate an I/O buffer from the controller memory buffer.
+ *
+ * \param ctrlr Controller from which to allocate memory buffer.
+ * \param size Size of buffer to allocate in bytes.
+ *
+ * \return Pointer to controller memory buffer allocation, or NULL if allocation was not possible.
+ */
+void *spdk_nvme_ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size);
+
+/**
+ * \brief Free a controller memory I/O buffer.
+ *
+ * \param ctrlr Controller from which the buffer was allocated.
+ * \param buf Buffer previously allocated by spdk_nvme_ctrlr_alloc_cmb_io_buffer().
+ * \param size Size of buf in bytes.
+ */
+void spdk_nvme_ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size);
+
 /**
  * \brief Get the identify namespace data as defined by the NVMe specification.
  *

lib/nvme/nvme_ctrlr.c

@@ -2060,3 +2060,29 @@ spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload, ui
 
 	return spdk_nvme_ctrlr_reset(ctrlr);
 }
+
+void *
+spdk_nvme_ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size)
+{
+	void *buf;
+
+	if (size == 0) {
+		return NULL;
+	}
+
+	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	buf = nvme_transport_ctrlr_alloc_cmb_io_buffer(ctrlr, size);
+	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return buf;
+}
+
+void
+spdk_nvme_ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size)
+{
+	if (buf && size) {
+		nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+		nvme_transport_ctrlr_free_cmb_io_buffer(ctrlr, buf, size);
+		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
+	}
+}

lib/nvme/nvme_internal.h

@@ -630,6 +630,8 @@ struct spdk_nvme_ctrlr *spdk_nvme_get_ctrlr_by_trid_unsafe(
 	uint32_t nvme_ ## name ## _ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr); \
 	uint16_t nvme_ ## name ## _ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr); \
 	struct spdk_nvme_qpair *nvme_ ## name ## _ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts); \
+	void *nvme_ ## name ## _ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size); \
+	int nvme_ ## name ## _ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size); \
 	int nvme_ ## name ## _ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair); \
 	int nvme_ ## name ## _ctrlr_reinit_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair); \
 	int nvme_ ## name ## _qpair_enable(struct spdk_nvme_qpair *qpair); \

lib/nvme/nvme_pcie.c

@@ -91,6 +91,8 @@ struct nvme_pcie_ctrlr {
 	void *cmb_mem_register_addr;
 	size_t cmb_mem_register_size;
 
+	bool cmb_io_data_supported;
+
 	/** stride in uint32_t units between doorbell registers (1 = 4 bytes, 2 = 8 bytes, ...) */
 	uint32_t doorbell_stride_u32;
 
@@ -515,7 +517,7 @@ nvme_pcie_ctrlr_map_cmb(struct nvme_pcie_ctrlr *pctrlr)
 		goto exit;
 	}
 	pctrlr->cmb_current_offset = mem_register_start - ((uint64_t)pctrlr->cmb_bar_virt_addr + offset);
-
+	pctrlr->cmb_io_data_supported = true;
 
 	return;
 exit:
@@ -565,6 +567,42 @@ nvme_pcie_ctrlr_alloc_cmb(struct spdk_nvme_ctrlr *ctrlr, uint64_t length, uint64
 	return 0;
 }
 
+void *
+nvme_pcie_ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size)
+{
+	struct nvme_pcie_ctrlr *pctrlr = nvme_pcie_ctrlr(ctrlr);
+	uint64_t offset;
+
+	if (pctrlr->cmb_bar_virt_addr == NULL) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "CMB not available\n");
+		return NULL;
+	}
+
+	if (!pctrlr->cmb_io_data_supported) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "CMB doesn't support I/O data\n");
+		return NULL;
+	}
+
+	if (nvme_pcie_ctrlr_alloc_cmb(ctrlr, size, 4, &offset) != 0) {
+		SPDK_DEBUGLOG(SPDK_LOG_NVME, "%zu-byte CMB allocation failed\n", size);
+		return NULL;
+	}
+
+	return pctrlr->cmb_bar_virt_addr + offset;
+}
+
+int
+nvme_pcie_ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size)
+{
+	/*
+	 * Do nothing for now.
+	 * TODO: Track free space so buffers may be reused.
+	 */
+	SPDK_ERRLOG("%s: no deallocation for CMB buffers yet!\n",
+		    __func__);
+	return 0;
+}
+
 static int
 nvme_pcie_ctrlr_allocate_bars(struct nvme_pcie_ctrlr *pctrlr)
 {

lib/nvme/nvme_rdma.c

@@ -1566,3 +1566,15 @@ nvme_rdma_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
 	 */
 	return 1;
 }
+
+void *
+nvme_rdma_ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size)
+{
+	return NULL;
+}
+
+int
+nvme_rdma_ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size)
+{
+	return 0;
+}

lib/nvme/nvme_transport.c

@@ -146,6 +146,18 @@ nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
 	NVME_TRANSPORT_CALL(ctrlr->trid.trtype, ctrlr_get_max_sges, (ctrlr));
 }
 
+void *
+nvme_transport_ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size)
+{
+	NVME_TRANSPORT_CALL(ctrlr->trid.trtype, ctrlr_alloc_cmb_io_buffer, (ctrlr, size));
+}
+
+int
+nvme_transport_ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size)
+{
+	NVME_TRANSPORT_CALL(ctrlr->trid.trtype, ctrlr_free_cmb_io_buffer, (ctrlr, buf, size));
+}
+
 struct spdk_nvme_qpair *
 nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
 				     const struct spdk_nvme_io_qpair_opts *opts)

test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c

@@ -127,6 +127,18 @@ nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
 	return 1;
 }
 
+void *
+nvme_transport_ctrlr_alloc_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, size_t size)
+{
+	return NULL;
+}
+
+int
+nvme_transport_ctrlr_free_cmb_io_buffer(struct spdk_nvme_ctrlr *ctrlr, void *buf, size_t size)
+{
+	return 0;
+}
+
 struct spdk_nvme_qpair *
 nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
 				     const struct spdk_nvme_io_qpair_opts *opts)