SPDK: Add end-to-end data protection support in driver

For those controllers which can support end-to-end data protection
feature, add the support in the driver layer.

Change-Id: Ifac3dd89dec9860773c850416a6116113a6ce22a
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>

include/spdk/nvme.h

@@ -533,6 +533,30 @@ uint64_t spdk_nvme_ns_get_num_sectors(struct spdk_nvme_ns *ns);
  */
 uint64_t spdk_nvme_ns_get_size(struct spdk_nvme_ns *ns);
 
+/**
+ * \brief Get the end-to-end data protection information type of the given namespace.
+ *
+ * This function is thread safe and can be called at any point while the controller is attached to
+ *  the SPDK NVMe driver.
+ */
+enum spdk_nvme_pi_type spdk_nvme_ns_get_pi_type(struct spdk_nvme_ns *ns);
+
+/**
+ * \brief Get the metadata size, in bytes, of the given namespace.
+ *
+ * This function is thread safe and can be called at any point while the controller is attached to
+ *  the SPDK NVMe driver.
+ */
+uint32_t spdk_nvme_ns_get_md_size(struct spdk_nvme_ns *ns);
+
+/**
+ * \brief True if the namespace can support extended LBA when end-to-end data protection enabled.
+ *
+ * This function is thread safe and can be called at any point while the controller is attached to
+ *  the SPDK NVMe driver.
+ */
+bool spdk_nvme_ns_supports_extended_lba(struct spdk_nvme_ns *ns);
+
 /**
  * \brief Namespace command support flags.
  */
@@ -541,6 +565,10 @@ enum spdk_nvme_ns_flags {
 	SPDK_NVME_NS_FLUSH_SUPPORTED		= 0x2, /**< The flush command is supported */
 	SPDK_NVME_NS_RESERVATION_SUPPORTED	= 0x4, /**< The reservation command is supported */
 	SPDK_NVME_NS_WRITE_ZEROES_SUPPORTED	= 0x8, /**< The write zeroes command is supported */
+	SPDK_NVME_NS_DPS_PI_SUPPORTED		= 0x10, /**< The end-to-end data protection is supported */
+	SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED	= 0x20, /**< The extended lba format is supported,
+							      metadata is transferred as a contiguous
+							      part of the logical block that it is associated with */
 };
 
 /**
@@ -619,6 +647,35 @@ int spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpa
 			    spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
 			    spdk_nvme_req_next_sge_cb next_sge_fn);
 
+/**
+ * \brief Submits a write I/O to the specified NVMe namespace.
+ *
+ * \param ns NVMe namespace to submit the write I/O
+ * \param qpair I/O queue pair to submit the request
+ * \param payload virtual address pointer to the data payload
+ * \param metadata virtual address pointer to the metadata payload, the length
+ *	           of metadata is specified by spdk_nvme_ns_get_md_size()
+ * \param lba starting LBA to write the data
+ * \param lba_count length (in sectors) for the write operation
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ * \param io_flags set flags, defined by the SPDK_NVME_IO_FLAGS_* entries
+ * 			in spdk/nvme_spec.h, for this I/O.
+ * \param apptag_mask application tag mask.
+ * \param apptag application tag to use end-to-end protection information.
+ *
+ * \return 0 if successfully submitted, ENOMEM if an nvme_request
+ *	     structure cannot be allocated for the I/O request
+ *
+ * The command is submitted to a qpair allocated by spdk_nvme_ctrlr_alloc_io_qpair().
+ * The user must ensure that only one thread submits I/O on a given qpair at any given time.
+ */
+int spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				   void *payload, void *metadata,
+				   uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn,
+				   void *cb_arg, uint32_t io_flags,
+				   uint16_t apptag_mask, uint16_t apptag);
+
 /**
  * \brief Submits a write zeroes I/O to the specified NVMe namespace.
  *
@@ -690,6 +747,34 @@ int spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpai
 			   spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
 			   spdk_nvme_req_next_sge_cb next_sge_fn);
 
+/**
+ * \brief Submits a read I/O to the specified NVMe namespace.
+ *
+ * \param ns NVMe namespace to submit the read I/O
+ * \param qpair I/O queue pair to submit the request
+ * \param payload virtual address pointer to the data payload
+ * \param metadata virtual address pointer to the metadata payload, the length
+ *	           of metadata is specified by spdk_nvme_ns_get_md_size()
+ * \param lba starting LBA to read the data
+ * \param lba_count length (in sectors) for the read operation
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ * \param io_flags set flags, defined in nvme_spec.h, for this I/O
+ * \param apptag_mask application tag mask.
+ * \param apptag application tag to use end-to-end protection information.
+ *
+ * \return 0 if successfully submitted, ENOMEM if an nvme_request
+ *	     structure cannot be allocated for the I/O request
+ *
+ * The command is submitted to a qpair allocated by spdk_nvme_ctrlr_alloc_io_qpair().
+ * The user must ensure that only one thread submits I/O on a given qpair at any given time.
+ */
+int spdk_nvme_ns_cmd_read_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				  void *payload, void *metadata,
+				  uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn,
+				  void *cb_arg, uint32_t io_flags,
+				  uint16_t apptag_mask, uint16_t apptag);
+
 /**
  * \brief Submits a deallocation request to the specified NVMe namespace.
  *

include/spdk/nvme_spec.h

@@ -1270,9 +1270,24 @@ struct spdk_nvme_format {
 };
 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_format) == 4, "Incorrect size");
 
+struct spdk_nvme_protection_info {
+	uint16_t	guard;
+	uint16_t	app_tag;
+	uint32_t	ref_tag;
+};
+SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_protection_info) == 8, "Incorrect size");
+
 #define spdk_nvme_cpl_is_error(cpl)					\
 	((cpl)->status.sc != 0 || (cpl)->status.sct != 0)
 
+/** Enable protection information checking of the Logical Block Reference Tag field */
+#define SPDK_NVME_IO_FLAGS_PRCHK_REFTAG (1U << 26)
+/** Enable protection information checking of the Application Tag field */
+#define SPDK_NVME_IO_FLAGS_PRCHK_APPTAG (1U << 27)
+/** Enable protection information checking of the Guard field */
+#define SPDK_NVME_IO_FLAGS_PRCHK_GUARD (1U << 28)
+/** The protection information is stripped or inserted when set this bit */
+#define SPDK_NVME_IO_FLAGS_PRACT (1U << 29)
 #define SPDK_NVME_IO_FLAGS_FORCE_UNIT_ACCESS (1U << 30)
 #define SPDK_NVME_IO_FLAGS_LIMITED_RETRY (1U << 31)
 

lib/nvme/nvme.c

@@ -136,6 +136,7 @@ nvme_allocate_request_contig(void *buffer, uint32_t payload_size, spdk_nvme_cmd_
 
 	payload.type = NVME_PAYLOAD_TYPE_CONTIG;
 	payload.u.contig = buffer;
+	payload.md = NULL;
 
 	return nvme_allocate_request(&payload, payload_size, cb_fn, cb_arg);
 }

lib/nvme/nvme_internal.h

@@ -161,6 +161,9 @@ struct __attribute__((packed)) nvme_payload {
 		} sgl;
 	} u;
 
+	/** Virtual memory address of a single physically contiguous metadata buffer */
+	void *md;
+
 	/** \ref nvme_payload_type */
 	uint8_t type;
 };
@@ -187,6 +190,7 @@ struct nvme_request {
 	 * This is used for I/O commands that are split into multiple requests.
 	 */
 	uint32_t			payload_offset;
+	uint32_t			md_offset;
 
 	spdk_nvme_cmd_cb		cb_fn;
 	void				*cb_arg;
@@ -320,6 +324,8 @@ struct spdk_nvme_ns {
 	struct spdk_nvme_ctrlr		*ctrlr;
 	uint32_t			stripe_size;
 	uint32_t			sector_size;
+	uint32_t			md_size;
+	uint32_t			pi_type;
 	uint32_t			sectors_per_max_io;
 	uint32_t			sectors_per_stripe;
 	uint16_t			id;

lib/nvme/nvme_ns.c

@@ -86,6 +86,15 @@ int nvme_ns_identify_update(struct spdk_nvme_ns *ns)
 	if (nsdata->nsrescap.raw) {
 		ns->flags |= SPDK_NVME_NS_RESERVATION_SUPPORTED;
 	}
+
+	ns->md_size = nsdata->lbaf[nsdata->flbas.format].ms;
+	ns->pi_type = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE;
+	if (nsdata->lbaf[nsdata->flbas.format].ms && nsdata->dps.pit) {
+		ns->flags |= SPDK_NVME_NS_DPS_PI_SUPPORTED;
+		ns->pi_type = nsdata->dps.pit;
+		if (nsdata->flbas.extended)
+			ns->flags |= SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED;
+	}
 	return rc;
 }
 
@@ -138,6 +147,23 @@ spdk_nvme_ns_get_flags(struct spdk_nvme_ns *ns)
 	return ns->flags;
 }
 
+enum spdk_nvme_pi_type
+spdk_nvme_ns_get_pi_type(struct spdk_nvme_ns *ns) {
+	return ns->pi_type;
+}
+
+bool
+spdk_nvme_ns_supports_extended_lba(struct spdk_nvme_ns *ns)
+{
+	return (ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) ? true : false;
+}
+
+uint32_t
+spdk_nvme_ns_get_md_size(struct spdk_nvme_ns *ns)
+{
+	return ns->md_size;
+}
+
 const struct spdk_nvme_ns_data *
 spdk_nvme_ns_get_data(struct spdk_nvme_ns *ns)
 {

lib/nvme/nvme_ns_cmd.c

@@ -36,7 +36,8 @@
 static struct nvme_request *_nvme_ns_cmd_rw(struct spdk_nvme_ns *ns,
 		const struct nvme_payload *payload, uint64_t lba,
 		uint32_t lba_count, spdk_nvme_cmd_cb cb_fn,
-		void *cb_arg, uint32_t opc, uint32_t io_flags);
+		void *cb_arg, uint32_t opc, uint32_t io_flags,
+		uint16_t apptag_mask, uint16_t apptag);
 
 static void
 nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl)
@@ -98,19 +99,28 @@ _nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns,
 			   uint64_t lba, uint32_t lba_count,
 			   spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
 			   uint32_t io_flags, struct nvme_request *req,
-			   uint32_t sectors_per_max_io, uint32_t sector_mask)
+			   uint32_t sectors_per_max_io, uint32_t sector_mask,
+			   uint16_t apptag_mask, uint16_t apptag)
 {
 	uint32_t		sector_size = ns->sector_size;
+	uint32_t		md_size = ns->md_size;
 	uint32_t		remaining_lba_count = lba_count;
 	uint32_t		offset = 0;
+	uint32_t		md_offset = 0;
 	struct nvme_request	*child, *tmp;
 
+	if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) {
+		/* for extended LBA only */
+		if ((ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) && !(io_flags & SPDK_NVME_IO_FLAGS_PRACT))
+			sector_size += ns->md_size;
+	}
+
 	while (remaining_lba_count > 0) {
 		lba_count = sectors_per_max_io - (lba & sector_mask);
 		lba_count = nvme_min(remaining_lba_count, lba_count);
 
 		child = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn,
-					cb_arg, opc, io_flags);
+					cb_arg, opc, io_flags, apptag_mask, apptag);
 		if (child == NULL) {
 			if (req->num_children) {
 				/* free all child nvme_request  */
@@ -123,10 +133,14 @@ _nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns,
 			return NULL;
 		}
 		child->payload_offset = offset;
+		/* for separate metadata buffer only */
+		if (payload->md)
+			child->md_offset = md_offset;
 		nvme_request_add_child(req, child);
 		remaining_lba_count -= lba_count;
 		lba += lba_count;
 		offset += lba_count * sector_size;
+		md_offset += lba_count * md_size;
 	}
 
 	return req;
@@ -135,7 +149,7 @@ _nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns,
 static struct nvme_request *
 _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload,
 		uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc,
-		uint32_t io_flags)
+		uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag)
 {
 	struct nvme_request	*req;
 	struct spdk_nvme_cmd	*cmd;
@@ -153,6 +167,12 @@ _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload,
 	sectors_per_max_io = ns->sectors_per_max_io;
 	sectors_per_stripe = ns->sectors_per_stripe;
 
+	if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) {
+		/* for extended LBA only */
+		if ((ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) && !(io_flags & SPDK_NVME_IO_FLAGS_PRACT))
+			sector_size += ns->md_size;
+	}
+
 	req = nvme_allocate_request(payload, lba_count * sector_size, cb_fn, cb_arg);
 	if (req == NULL) {
 		return NULL;
@@ -168,10 +188,10 @@ _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload,
 	    (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) {
 
 		return _nvme_ns_cmd_split_request(ns, payload, lba, lba_count, cb_fn, cb_arg, opc,
-						  io_flags, req, sectors_per_stripe, sectors_per_stripe - 1);
+						  io_flags, req, sectors_per_stripe, sectors_per_stripe - 1, apptag_mask, apptag);
 	} else if (lba_count > sectors_per_max_io) {
 		return _nvme_ns_cmd_split_request(ns, payload, lba, lba_count, cb_fn, cb_arg, opc,
-						  io_flags, req, sectors_per_max_io, 0);
+						  io_flags, req, sectors_per_max_io, 0, apptag_mask, apptag);
 	} else {
 		cmd = &req->cmd;
 		cmd->opc = opc;
@@ -180,8 +200,20 @@ _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload,
 		tmp_lba = (uint64_t *)&cmd->cdw10;
 		*tmp_lba = lba;
 
+		if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) {
+			switch (ns->pi_type) {
+			case SPDK_NVME_FMT_NVM_PROTECTION_TYPE1:
+			case SPDK_NVME_FMT_NVM_PROTECTION_TYPE2:
+				cmd->cdw14 = (uint32_t)lba;
+				break;
+			}
+		}
+
 		cmd->cdw12 = lba_count - 1;
 		cmd->cdw12 |= io_flags;
+
+		cmd->cdw15 = apptag_mask;
+		cmd->cdw15 = (cmd->cdw15 << 16 | apptag);
 	}
 
 	return req;
@@ -198,8 +230,33 @@ spdk_nvme_ns_cmd_read(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, vo
 
 	payload.type = NVME_PAYLOAD_TYPE_CONTIG;
 	payload.u.contig = buffer;
+	payload.md = NULL;
 
-	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags);
+	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags, 0,
+			      0);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else {
+		return ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_read_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer,
+			      void *metadata,
+			      uint64_t lba,
+			      uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+			      uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	payload.type = NVME_PAYLOAD_TYPE_CONTIG;
+	payload.u.contig = buffer;
+	payload.md = metadata;
+
+	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags,
+			      apptag_mask, apptag);
 	if (req != NULL) {
 		return nvme_qpair_submit_request(qpair, req);
 	} else {
@@ -221,11 +278,13 @@ spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
 		return EINVAL;
 
 	payload.type = NVME_PAYLOAD_TYPE_SGL;
+	payload.md = NULL;
 	payload.u.sgl.reset_sgl_fn = reset_sgl_fn;
 	payload.u.sgl.next_sge_fn = next_sge_fn;
 	payload.u.sgl.cb_arg = cb_arg;
 
-	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags);
+	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags, 0,
+			      0);
 	if (req != NULL) {
 		return nvme_qpair_submit_request(qpair, req);
 	} else {
@@ -244,8 +303,32 @@ spdk_nvme_ns_cmd_write(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
 
 	payload.type = NVME_PAYLOAD_TYPE_CONTIG;
 	payload.u.contig = buffer;
+	payload.md = NULL;
 
-	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags);
+	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags, 0,
+			      0);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else {
+		return ENOMEM;
+	}
+}
+
+int
+spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			       void *buffer, void *metadata, uint64_t lba,
+			       uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,
+			       uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	payload.type = NVME_PAYLOAD_TYPE_CONTIG;
+	payload.u.contig = buffer;
+	payload.md = metadata;
+
+	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags,
+			      apptag_mask, apptag);
 	if (req != NULL) {
 		return nvme_qpair_submit_request(qpair, req);
 	} else {
@@ -267,11 +350,13 @@ spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
 		return EINVAL;
 
 	payload.type = NVME_PAYLOAD_TYPE_SGL;
+	payload.md = NULL;
 	payload.u.sgl.reset_sgl_fn = reset_sgl_fn;
 	payload.u.sgl.next_sge_fn = next_sge_fn;
 	payload.u.sgl.cb_arg = cb_arg;
 
-	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags);
+	req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags, 0,
+			      0);
 	if (req != NULL) {
 		return nvme_qpair_submit_request(qpair, req);
 	} else {

lib/nvme/nvme_qpair.c

@@ -612,6 +612,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ
 	uint64_t phys_addr;
 	void *seg_addr;
 	uint32_t nseg, cur_nseg, modulo, unaligned;
+	void *md_payload;
 	void *payload = req->payload.u.contig + req->payload_offset;
 
 	phys_addr = nvme_vtophys(payload);
@@ -626,6 +627,15 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ
 		nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE));
 	}
 
+	if (req->payload.md) {
+		md_payload = req->payload.md + req->md_offset;
+		tr->req->cmd.mptr = nvme_vtophys(md_payload);
+		if (tr->req->cmd.mptr == NVME_VTOPHYS_ERROR) {
+			_nvme_fail_request_bad_vtophys(qpair, tr);
+			return -1;
+		}
+	}
+
 	tr->req->cmd.psdt = SPDK_NVME_PSDT_PRP;
 	tr->req->cmd.dptr.prp.prp1 = phys_addr;
 	if (nseg == 2) {

test/lib/nvme/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(CURDIR)/../../..
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y = unit aer reset sgl
+DIRS-y = unit aer reset sgl e2edp
 
 .PHONY: all clean $(DIRS-y)
 

test/lib/nvme/e2edp/.gitignore

@@ -0,0 +1 @@
+nvme_dp

test/lib/nvme/e2edp/Makefile

@@ -0,0 +1,57 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(CURDIR)/../../../..
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+APP = nvme_dp
+
+C_SRCS := nvme_dp.c
+
+CFLAGS += -I. $(DPDK_INC)
+
+SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
+	     $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
+	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a
+
+LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) -lpthread $(DPDK_LIB) -lrt
+
+all : $(APP)
+
+$(APP) : $(OBJS) $(SPDK_LIBS)
+	$(LINK_C)
+
+clean :
+	$(CLEAN_C) $(APP)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

test/lib/nvme/e2edp/nvme_dp.c

@@ -0,0 +1,626 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NVMe end-to-end data protection test
+ */
+
+#include <stdbool.h>
+#include <inttypes.h>
+#include <string.h>
+
+#include <rte_config.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_lcore.h>
+
+#include "spdk/nvme.h"
+#include "spdk/pci.h"
+
+static uint32_t swap32(uint32_t value)
+{
+	uint32_t result = 0;
+	result |= (value & 0x000000FF) << 24;
+	result |= (value & 0x0000FF00) << 8;
+	result |= (value & 0x00FF0000) >> 8;
+	result |= (value & 0xFF000000) >> 24;
+	return result;
+}
+
+
+static uint16_t swap16(uint16_t value)
+{
+	uint16_t result = 0;
+
+	result |= (value & 0x00FF) << 8;
+	result |= (value & 0xFF00) >> 8;
+
+	return result;
+}
+
+struct rte_mempool *request_mempool;
+
+#define MAX_DEVS 64
+
+#define DATA_PATTERN 0x5A
+
+struct dev {
+	struct spdk_nvme_ctrlr			*ctrlr;
+	char 					name[100];
+};
+
+static struct dev devs[MAX_DEVS];
+static int num_devs = 0;
+
+#define foreach_dev(iter) \
+	for (iter = devs; iter - devs < num_devs; iter++)
+
+static int io_complete_flag = 0;
+
+struct io_request {
+	void *contig;
+	void *metadata;
+	bool use_extended_lba;
+	uint64_t lba;
+	uint32_t lba_count;
+	uint16_t apptag_mask;
+	uint16_t apptag;
+};
+
+static void
+io_complete(void *ctx, const struct spdk_nvme_cpl *cpl)
+{
+	if (spdk_nvme_cpl_is_error(cpl))
+		io_complete_flag = 2;
+	else
+		io_complete_flag = 1;
+}
+
+/*
+ * No protection information with PRACT setting to 1,
+ *  both extended LBA format and separate metadata can
+ *  run the test case.
+ */
+static uint32_t dp_with_pract_test(struct spdk_nvme_ns *ns, struct io_request *req,
+				   uint32_t *io_flags)
+{
+	uint32_t sector_size;
+
+	req->lba_count = 8;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);
+	/* No additional metadata buffer provided */
+	req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000);
+	if (!req->contig)
+		return 0;
+
+	switch (spdk_nvme_ns_get_pi_type(ns)) {
+	case SPDK_NVME_FMT_NVM_PROTECTION_TYPE3:
+		*io_flags = SPDK_NVME_IO_FLAGS_PRCHK_GUARD | SPDK_NVME_IO_FLAGS_PRACT;
+		break;
+	case SPDK_NVME_FMT_NVM_PROTECTION_TYPE1:
+	case SPDK_NVME_FMT_NVM_PROTECTION_TYPE2:
+		*io_flags = SPDK_NVME_IO_FLAGS_PRCHK_GUARD | SPDK_NVME_IO_FLAGS_PRCHK_REFTAG |
+			    SPDK_NVME_IO_FLAGS_PRACT;
+		break;
+	default:
+		*io_flags = 0;
+		break;
+	}
+	req->lba = 0x100000;
+	req->use_extended_lba = false;
+	req->metadata = NULL;
+
+	return req->lba_count;
+}
+
+/* Block Reference Tag checked for TYPE1 and TYPE2 with PRACT setting to 0 */
+static uint32_t dp_without_pract_extended_lba_test(struct spdk_nvme_ns *ns, struct io_request *req,
+		uint32_t *io_flags)
+{
+	struct spdk_nvme_protection_info *pi;
+	uint32_t md_size, sector_size;
+
+	req->lba_count = 2;
+
+	switch (spdk_nvme_ns_get_pi_type(ns)) {
+	case SPDK_NVME_FMT_NVM_PROTECTION_TYPE3:
+		return 0;
+	default:
+		break;
+	}
+
+	/* extended LBA only for the test case */
+	if (!(spdk_nvme_ns_supports_extended_lba(ns)))
+		return 0;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);;
+	md_size = spdk_nvme_ns_get_md_size(ns);
+	req->contig = rte_zmalloc(NULL, (sector_size + md_size) * req->lba_count, 0x1000);
+	if (!req->contig)
+		return 0;
+
+	req->lba = 0x200000;
+	req->use_extended_lba = true;
+	req->metadata = NULL;
+	pi = (struct spdk_nvme_protection_info *)(req->contig + sector_size + md_size - 8);
+	/* big-endian for reference tag */
+	pi->ref_tag = swap32((uint32_t)req->lba);
+
+	pi = (struct spdk_nvme_protection_info *)(req->contig + (sector_size + md_size) * 2 - 8);
+	/* is incremented for each subsequent logical block */
+	pi->ref_tag = swap32((uint32_t)req->lba + 1);
+
+	*io_flags = SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
+
+	return req->lba_count;
+}
+
+/* LBA + Metadata without data protection bits setting */
+static uint32_t dp_without_flags_extended_lba_test(struct spdk_nvme_ns *ns, struct io_request *req,
+		uint32_t *io_flags)
+{
+	uint32_t md_size, sector_size;
+
+	req->lba_count = 16;
+
+	/* extended LBA only for the test case */
+	if (!(spdk_nvme_ns_supports_extended_lba(ns)))
+		return 0;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);;
+	md_size = spdk_nvme_ns_get_md_size(ns);
+	req->contig = rte_zmalloc(NULL, (sector_size + md_size) * req->lba_count, 0x1000);
+	if (!req->contig)
+		return 0;
+
+	req->lba = 0x400000;
+	req->use_extended_lba = true;
+	req->metadata = NULL;
+	*io_flags = 0;
+
+	return req->lba_count;
+}
+
+/* Block Reference Tag checked for TYPE1 and TYPE2 with PRACT setting to 0 */
+static uint32_t dp_without_pract_separate_meta_test(struct spdk_nvme_ns *ns, struct io_request *req,
+		uint32_t *io_flags)
+{
+	struct spdk_nvme_protection_info *pi;
+	uint32_t md_size, sector_size;
+
+	req->lba_count = 2;
+
+	switch (spdk_nvme_ns_get_pi_type(ns)) {
+	case SPDK_NVME_FMT_NVM_PROTECTION_TYPE3:
+		return 0;
+	default:
+		break;
+	}
+
+	/* separate metadata payload for the test case */
+	if (spdk_nvme_ns_supports_extended_lba(ns))
+		return 0;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);;
+	md_size = spdk_nvme_ns_get_md_size(ns);
+	req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000);
+	if (!req->contig)
+		return 0;
+
+	req->metadata = rte_zmalloc(NULL, md_size * req->lba_count, 0x1000);
+	if (!req->metadata) {
+		rte_free(req->contig);
+		return 0;
+	}
+
+	req->lba = 0x400000;
+	req->use_extended_lba = false;
+
+	/* last 8 bytes if the metadata size bigger than 8 */
+	pi = (struct spdk_nvme_protection_info *)(req->metadata + md_size - 8);
+	/* big-endian for reference tag */
+	pi->ref_tag = swap32((uint32_t)req->lba);
+
+	pi = (struct spdk_nvme_protection_info *)(req->metadata + md_size * 2 - 8);
+	/* is incremented for each subsequent logical block */
+	pi->ref_tag = swap32((uint32_t)req->lba + 1);
+
+	*io_flags = SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
+
+	return req->lba_count;
+}
+
+/* Application Tag checked with PRACT setting to 0 */
+static uint32_t dp_without_pract_separate_meta_apptag_test(struct spdk_nvme_ns *ns,
+		struct io_request *req,
+		uint32_t *io_flags)
+{
+	struct spdk_nvme_protection_info *pi;
+	uint32_t md_size, sector_size;
+
+	req->lba_count = 1;
+
+	/* separate metadata payload for the test case */
+	if (spdk_nvme_ns_supports_extended_lba(ns))
+		return 0;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);;
+	md_size = spdk_nvme_ns_get_md_size(ns);
+	req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000);
+	if (!req->contig)
+		return 0;
+
+	req->metadata = rte_zmalloc(NULL, md_size * req->lba_count, 0x1000);
+	if (!req->metadata) {
+		rte_free(req->contig);
+		return 0;
+	}
+
+	req->lba = 0x500000;
+	req->use_extended_lba = false;
+	req->apptag_mask = 0xFFFF;
+	req->apptag = req->lba_count;
+
+	/* last 8 bytes if the metadata size bigger than 8 */
+	pi = (struct spdk_nvme_protection_info *)(req->metadata + md_size - 8);
+	pi->app_tag = swap16(req->lba_count);
+
+	*io_flags = SPDK_NVME_IO_FLAGS_PRCHK_APPTAG;
+
+	return req->lba_count;
+}
+
+/*
+ * LBA + Metadata without data protection bits setting,
+ *  separate metadata payload for the test case.
+ */
+static uint32_t dp_without_flags_separate_meta_test(struct spdk_nvme_ns *ns, struct io_request *req,
+		uint32_t *io_flags)
+{
+	uint32_t md_size, sector_size;
+
+	req->lba_count = 16;
+
+	/* separate metadata payload for the test case */
+	if (spdk_nvme_ns_supports_extended_lba(ns))
+		return 0;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);;
+	md_size = spdk_nvme_ns_get_md_size(ns);
+	req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000);
+	if (!req->contig)
+		return 0;
+
+	req->metadata = rte_zmalloc(NULL, md_size * req->lba_count, 0x1000);
+	if (!req->metadata) {
+		rte_free(req->contig);
+		return 0;
+	}
+
+	req->lba = 0x600000;
+	req->use_extended_lba = false;
+	*io_flags = 0;
+
+	return req->lba_count;
+}
+
+typedef uint32_t (*nvme_build_io_req_fn_t)(struct spdk_nvme_ns *ns, struct io_request *req,
+		uint32_t *lba_count);
+
+static void
+free_req(struct io_request *req)
+{
+	if (req == NULL) {
+		return;
+	}
+
+	if (req->contig)
+		rte_free(req->contig);
+
+	if (req->metadata)
+		rte_free(req->metadata);
+
+	rte_free(req);
+}
+
+static void
+ns_data_buffer_reset(struct spdk_nvme_ns *ns, struct io_request *req, uint8_t data_pattern)
+{
+	uint32_t md_size, sector_size;
+	uint32_t i, offset = 0;
+	uint8_t *buf;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);
+	md_size = spdk_nvme_ns_get_md_size(ns);
+
+	for (i = 0; i < req->lba_count; i++) {
+		if (req->use_extended_lba)
+			offset = (sector_size + md_size) * i;
+		else
+			offset = sector_size * i;
+
+		buf = (uint8_t *)req->contig + offset;
+		memset(buf, data_pattern, sector_size);
+	}
+}
+
+static int
+ns_data_buffer_compare(struct spdk_nvme_ns *ns, struct io_request *req, uint8_t data_pattern)
+{
+	uint32_t md_size, sector_size;
+	uint32_t i, j, offset = 0;
+	uint8_t *buf;
+
+	sector_size = spdk_nvme_ns_get_sector_size(ns);
+	md_size = spdk_nvme_ns_get_md_size(ns);
+
+	for (i = 0; i < req->lba_count; i++) {
+		if (req->use_extended_lba)
+			offset = (sector_size + md_size) * i;
+		else
+			offset = sector_size * i;
+
+		buf = (uint8_t *)req->contig + offset;
+		for (j = 0; j < sector_size; j++) {
+			if (buf[j] != data_pattern) {
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int
+write_read_e2e_dp_tests(struct dev *dev, nvme_build_io_req_fn_t build_io_fn, const char *test_name)
+{
+	int rc = 0;
+	uint32_t lba_count;
+	uint32_t io_flags = 0;
+
+	struct io_request *req;
+	struct spdk_nvme_ns *ns;
+	struct spdk_nvme_qpair *qpair;
+	const struct spdk_nvme_ns_data *nsdata;
+
+	ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, 1);
+	if (!ns) {
+		fprintf(stderr, "Null namespace\n");
+		return 0;
+	}
+
+	if (!(spdk_nvme_ns_get_flags(ns) & SPDK_NVME_NS_DPS_PI_SUPPORTED))
+		return 0;
+
+	nsdata = spdk_nvme_ns_get_data(ns);
+	if (!nsdata || !spdk_nvme_ns_get_sector_size(ns)) {
+		fprintf(stderr, "Empty nsdata or wrong sector size\n");
+		return 0;
+	}
+
+	req = rte_zmalloc(NULL, sizeof(*req), 0);
+	if (!req) {
+		fprintf(stderr, "Allocate request failed\n");
+		return 0;
+	}
+
+	/* IO parameters setting */
+	lba_count = build_io_fn(ns, req, &io_flags);
+
+	if (!lba_count) {
+		fprintf(stderr, "%s: %s bypass the test case\n", dev->name, test_name);
+		free_req(req);
+		return 0;
+	}
+
+	qpair = spdk_nvme_ctrlr_alloc_io_qpair(dev->ctrlr, 0);
+	if (!qpair) {
+		free_req(req);
+		return -1;
+	}
+
+	ns_data_buffer_reset(ns, req, DATA_PATTERN);
+	if (req->use_extended_lba)
+		rc = spdk_nvme_ns_cmd_write(ns, qpair, req->contig, req->lba, lba_count,
+					    io_complete, req, io_flags);
+	else
+		rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, req->contig, req->metadata, req->lba, lba_count,
+						    io_complete, req, io_flags, req->apptag_mask, req->apptag);
+
+	if (rc != 0) {
+		fprintf(stderr, "%s: %s write submit failed\n", dev->name, test_name);
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+		free_req(req);
+		return -1;
+	}
+
+	io_complete_flag = 0;
+
+	while (!io_complete_flag)
+		spdk_nvme_qpair_process_completions(qpair, 1);
+
+	if (io_complete_flag != 1) {
+		fprintf(stderr, "%s: %s write exec failed\n", dev->name, test_name);
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+		free_req(req);
+		return -1;
+	}
+
+	/* reset completion flag */
+	io_complete_flag = 0;
+
+	ns_data_buffer_reset(ns, req, 0);
+	if (req->use_extended_lba)
+		rc = spdk_nvme_ns_cmd_read(ns, qpair, req->contig, req->lba, lba_count,
+					   io_complete, req, io_flags);
+	else
+		rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, req->contig, req->metadata, req->lba, lba_count,
+						   io_complete, req, io_flags, req->apptag_mask, req->apptag);
+
+	if (rc != 0) {
+		fprintf(stderr, "%s: %s read failed\n", dev->name, test_name);
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+		free_req(req);
+		return -1;
+	}
+
+	while (!io_complete_flag)
+		spdk_nvme_qpair_process_completions(qpair, 1);
+
+	if (io_complete_flag != 1) {
+		fprintf(stderr, "%s: %s read failed\n", dev->name, test_name);
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+		free_req(req);
+		return -1;
+	}
+
+	rc = ns_data_buffer_compare(ns, req, DATA_PATTERN);
+	if (rc < 0) {
+		fprintf(stderr, "%s: %s write/read success, but memcmp Failed\n", dev->name, test_name);
+		spdk_nvme_ctrlr_free_io_qpair(qpair);
+		free_req(req);
+		return -1;
+	}
+
+	fprintf(stdout, "%s: %s test passed\n", dev->name, test_name);
+	spdk_nvme_ctrlr_free_io_qpair(qpair);
+	free_req(req);
+	return rc;
+}
+
+static bool
+probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
+{
+	if (spdk_pci_device_has_non_uio_driver(dev)) {
+		fprintf(stderr, "non-uio kernel driver attached to NVMe\n");
+		fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n",
+			spdk_pci_device_get_domain(dev),
+			spdk_pci_device_get_bus(dev),
+			spdk_pci_device_get_dev(dev),
+			spdk_pci_device_get_func(dev));
+		fprintf(stderr, " skipping...\n");
+		return false;
+	}
+
+	printf("Attaching to %04x:%02x:%02x.%02x\n",
+	       spdk_pci_device_get_domain(dev),
+	       spdk_pci_device_get_bus(dev),
+	       spdk_pci_device_get_dev(dev),
+	       spdk_pci_device_get_func(dev));
+
+	return true;
+}
+
+static void
+attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_nvme_ctrlr *ctrlr,
+	  const struct spdk_nvme_ctrlr_opts *opts)
+{
+	struct dev *dev;
+
+	/* add to dev list */
+	dev = &devs[num_devs++];
+
+	dev->ctrlr = ctrlr;
+
+	snprintf(dev->name, sizeof(dev->name), "%04X:%02X:%02X.%02X",
+		 spdk_pci_device_get_domain(pci_dev),
+		 spdk_pci_device_get_bus(pci_dev),
+		 spdk_pci_device_get_dev(pci_dev),
+		 spdk_pci_device_get_func(pci_dev));
+
+	printf("Attached to %s\n", dev->name);
+}
+
+
+static const char *ealargs[] = {
+	"nvme_dp",
+	"-c 0x1",
+	"-n 4",
+};
+
+int main(int argc, char **argv)
+{
+	struct dev			*iter;
+	int				rc, i;
+
+	printf("NVMe Write/Read with End-to-End data protection test\n");
+
+	rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]),
+			  (char **)(void *)(uintptr_t)ealargs);
+
+	if (rc < 0) {
+		fprintf(stderr, "could not initialize dpdk\n");
+		exit(1);
+	}
+
+	request_mempool = rte_mempool_create("nvme_request", 8192,
+					     spdk_nvme_request_size(), 128, 0,
+					     NULL, NULL, NULL, NULL,
+					     SOCKET_ID_ANY, 0);
+
+	if (request_mempool == NULL) {
+		fprintf(stderr, "could not initialize request mempool\n");
+		exit(1);
+	}
+
+	if (spdk_nvme_probe(NULL, probe_cb, attach_cb) != 0) {
+		fprintf(stderr, "nvme_probe() failed\n");
+		exit(1);
+	}
+
+	rc = 0;
+	foreach_dev(iter) {
+#define TEST(x) write_read_e2e_dp_tests(iter, x, #x)
+		if (TEST(dp_with_pract_test)
+		    || TEST(dp_without_pract_extended_lba_test)
+		    || TEST(dp_without_flags_extended_lba_test)
+		    || TEST(dp_without_pract_separate_meta_test)
+		    || TEST(dp_without_pract_separate_meta_apptag_test)
+		    || TEST(dp_without_flags_separate_meta_test)) {
+#undef TEST
+			rc = 1;
+			printf("%s: failed End-to-End data protection tests\n", iter->name);
+		}
+	}
+
+	printf("Cleaning up...\n");
+
+	for (i = 0; i < num_devs; i++) {
+		struct dev *dev = &devs[i];
+
+		spdk_nvme_detach(dev->ctrlr);
+	}
+
+	return rc;
+}

test/lib/nvme/nvme.sh

@@ -43,4 +43,8 @@ timing_enter sgl
 $testdir/sgl/sgl
 timing_exit sgl
 
+timing_enter e2edp
+$testdir/e2edp/nvme_dp
+timing_exit e2edp
+
 timing_exit nvme