From 784182ed475ce0e95973a39de20e40c96cbe0943 Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Thu, 7 Apr 2016 14:52:43 +0800 Subject: [PATCH] SPDK: Add end-to-end data protection support in driver For those controllers which can support end-to-end data protection feature, add the support in the driver layer. Change-Id: Ifac3dd89dec9860773c850416a6116113a6ce22a Signed-off-by: Changpeng Liu --- include/spdk/nvme.h | 85 +++++ include/spdk/nvme_spec.h | 15 + lib/nvme/nvme.c | 1 + lib/nvme/nvme_internal.h | 6 + lib/nvme/nvme_ns.c | 26 ++ lib/nvme/nvme_ns_cmd.c | 105 +++++- lib/nvme/nvme_qpair.c | 10 + test/lib/nvme/Makefile | 2 +- test/lib/nvme/e2edp/.gitignore | 1 + test/lib/nvme/e2edp/Makefile | 57 +++ test/lib/nvme/e2edp/nvme_dp.c | 626 +++++++++++++++++++++++++++++++++ test/lib/nvme/nvme.sh | 4 + 12 files changed, 927 insertions(+), 11 deletions(-) create mode 100644 test/lib/nvme/e2edp/.gitignore create mode 100644 test/lib/nvme/e2edp/Makefile create mode 100644 test/lib/nvme/e2edp/nvme_dp.c diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index bb4019b1a..4212e330f 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -533,6 +533,30 @@ uint64_t spdk_nvme_ns_get_num_sectors(struct spdk_nvme_ns *ns); */ uint64_t spdk_nvme_ns_get_size(struct spdk_nvme_ns *ns); +/** + * \brief Get the end-to-end data protection information type of the given namespace. + * + * This function is thread safe and can be called at any point while the controller is attached to + * the SPDK NVMe driver. + */ +enum spdk_nvme_pi_type spdk_nvme_ns_get_pi_type(struct spdk_nvme_ns *ns); + +/** + * \brief Get the metadata size, in bytes, of the given namespace. + * + * This function is thread safe and can be called at any point while the controller is attached to + * the SPDK NVMe driver. + */ +uint32_t spdk_nvme_ns_get_md_size(struct spdk_nvme_ns *ns); + +/** + * \brief True if the namespace can support extended LBA when end-to-end data protection enabled. + * + * This function is thread safe and can be called at any point while the controller is attached to + * the SPDK NVMe driver. + */ +bool spdk_nvme_ns_supports_extended_lba(struct spdk_nvme_ns *ns); + /** * \brief Namespace command support flags. */ @@ -541,6 +565,10 @@ enum spdk_nvme_ns_flags { SPDK_NVME_NS_FLUSH_SUPPORTED = 0x2, /**< The flush command is supported */ SPDK_NVME_NS_RESERVATION_SUPPORTED = 0x4, /**< The reservation command is supported */ SPDK_NVME_NS_WRITE_ZEROES_SUPPORTED = 0x8, /**< The write zeroes command is supported */ + SPDK_NVME_NS_DPS_PI_SUPPORTED = 0x10, /**< The end-to-end data protection is supported */ + SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED = 0x20, /**< The extended lba format is supported, + metadata is transferred as a contiguous + part of the logical block that it is associated with */ }; /** @@ -619,6 +647,35 @@ int spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpa spdk_nvme_req_reset_sgl_cb reset_sgl_fn, spdk_nvme_req_next_sge_cb next_sge_fn); +/** + * \brief Submits a write I/O to the specified NVMe namespace. + * + * \param ns NVMe namespace to submit the write I/O + * \param qpair I/O queue pair to submit the request + * \param payload virtual address pointer to the data payload + * \param metadata virtual address pointer to the metadata payload, the length + * of metadata is specified by spdk_nvme_ns_get_md_size() + * \param lba starting LBA to write the data + * \param lba_count length (in sectors) for the write operation + * \param cb_fn callback function to invoke when the I/O is completed + * \param cb_arg argument to pass to the callback function + * \param io_flags set flags, defined by the SPDK_NVME_IO_FLAGS_* entries + * in spdk/nvme_spec.h, for this I/O. + * \param apptag_mask application tag mask. + * \param apptag application tag to use end-to-end protection information. + * + * \return 0 if successfully submitted, ENOMEM if an nvme_request + * structure cannot be allocated for the I/O request + * + * The command is submitted to a qpair allocated by spdk_nvme_ctrlr_alloc_io_qpair(). + * The user must ensure that only one thread submits I/O on a given qpair at any given time. + */ +int spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, + void *payload, void *metadata, + uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, + void *cb_arg, uint32_t io_flags, + uint16_t apptag_mask, uint16_t apptag); + /** * \brief Submits a write zeroes I/O to the specified NVMe namespace. * @@ -690,6 +747,34 @@ int spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpai spdk_nvme_req_reset_sgl_cb reset_sgl_fn, spdk_nvme_req_next_sge_cb next_sge_fn); +/** + * \brief Submits a read I/O to the specified NVMe namespace. + * + * \param ns NVMe namespace to submit the read I/O + * \param qpair I/O queue pair to submit the request + * \param payload virtual address pointer to the data payload + * \param metadata virtual address pointer to the metadata payload, the length + * of metadata is specified by spdk_nvme_ns_get_md_size() + * \param lba starting LBA to read the data + * \param lba_count length (in sectors) for the read operation + * \param cb_fn callback function to invoke when the I/O is completed + * \param cb_arg argument to pass to the callback function + * \param io_flags set flags, defined in nvme_spec.h, for this I/O + * \param apptag_mask application tag mask. + * \param apptag application tag to use end-to-end protection information. + * + * \return 0 if successfully submitted, ENOMEM if an nvme_request + * structure cannot be allocated for the I/O request + * + * The command is submitted to a qpair allocated by spdk_nvme_ctrlr_alloc_io_qpair(). + * The user must ensure that only one thread submits I/O on a given qpair at any given time. + */ +int spdk_nvme_ns_cmd_read_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, + void *payload, void *metadata, + uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, + void *cb_arg, uint32_t io_flags, + uint16_t apptag_mask, uint16_t apptag); + /** * \brief Submits a deallocation request to the specified NVMe namespace. * diff --git a/include/spdk/nvme_spec.h b/include/spdk/nvme_spec.h index d999f0f8d..3332064ce 100644 --- a/include/spdk/nvme_spec.h +++ b/include/spdk/nvme_spec.h @@ -1270,9 +1270,24 @@ struct spdk_nvme_format { }; SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_format) == 4, "Incorrect size"); +struct spdk_nvme_protection_info { + uint16_t guard; + uint16_t app_tag; + uint32_t ref_tag; +}; +SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_protection_info) == 8, "Incorrect size"); + #define spdk_nvme_cpl_is_error(cpl) \ ((cpl)->status.sc != 0 || (cpl)->status.sct != 0) +/** Enable protection information checking of the Logical Block Reference Tag field */ +#define SPDK_NVME_IO_FLAGS_PRCHK_REFTAG (1U << 26) +/** Enable protection information checking of the Application Tag field */ +#define SPDK_NVME_IO_FLAGS_PRCHK_APPTAG (1U << 27) +/** Enable protection information checking of the Guard field */ +#define SPDK_NVME_IO_FLAGS_PRCHK_GUARD (1U << 28) +/** The protection information is stripped or inserted when set this bit */ +#define SPDK_NVME_IO_FLAGS_PRACT (1U << 29) #define SPDK_NVME_IO_FLAGS_FORCE_UNIT_ACCESS (1U << 30) #define SPDK_NVME_IO_FLAGS_LIMITED_RETRY (1U << 31) diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c index 5ff9c7141..3260f128b 100644 --- a/lib/nvme/nvme.c +++ b/lib/nvme/nvme.c @@ -136,6 +136,7 @@ nvme_allocate_request_contig(void *buffer, uint32_t payload_size, spdk_nvme_cmd_ payload.type = NVME_PAYLOAD_TYPE_CONTIG; payload.u.contig = buffer; + payload.md = NULL; return nvme_allocate_request(&payload, payload_size, cb_fn, cb_arg); } diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index 9c109734e..88c137419 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -161,6 +161,9 @@ struct __attribute__((packed)) nvme_payload { } sgl; } u; + /** Virtual memory address of a single physically contiguous metadata buffer */ + void *md; + /** \ref nvme_payload_type */ uint8_t type; }; @@ -187,6 +190,7 @@ struct nvme_request { * This is used for I/O commands that are split into multiple requests. */ uint32_t payload_offset; + uint32_t md_offset; spdk_nvme_cmd_cb cb_fn; void *cb_arg; @@ -320,6 +324,8 @@ struct spdk_nvme_ns { struct spdk_nvme_ctrlr *ctrlr; uint32_t stripe_size; uint32_t sector_size; + uint32_t md_size; + uint32_t pi_type; uint32_t sectors_per_max_io; uint32_t sectors_per_stripe; uint16_t id; diff --git a/lib/nvme/nvme_ns.c b/lib/nvme/nvme_ns.c index b5267d91d..0ed715edb 100644 --- a/lib/nvme/nvme_ns.c +++ b/lib/nvme/nvme_ns.c @@ -86,6 +86,15 @@ int nvme_ns_identify_update(struct spdk_nvme_ns *ns) if (nsdata->nsrescap.raw) { ns->flags |= SPDK_NVME_NS_RESERVATION_SUPPORTED; } + + ns->md_size = nsdata->lbaf[nsdata->flbas.format].ms; + ns->pi_type = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE; + if (nsdata->lbaf[nsdata->flbas.format].ms && nsdata->dps.pit) { + ns->flags |= SPDK_NVME_NS_DPS_PI_SUPPORTED; + ns->pi_type = nsdata->dps.pit; + if (nsdata->flbas.extended) + ns->flags |= SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED; + } return rc; } @@ -138,6 +147,23 @@ spdk_nvme_ns_get_flags(struct spdk_nvme_ns *ns) return ns->flags; } +enum spdk_nvme_pi_type +spdk_nvme_ns_get_pi_type(struct spdk_nvme_ns *ns) { + return ns->pi_type; +} + +bool +spdk_nvme_ns_supports_extended_lba(struct spdk_nvme_ns *ns) +{ + return (ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) ? true : false; +} + +uint32_t +spdk_nvme_ns_get_md_size(struct spdk_nvme_ns *ns) +{ + return ns->md_size; +} + const struct spdk_nvme_ns_data * spdk_nvme_ns_get_data(struct spdk_nvme_ns *ns) { diff --git a/lib/nvme/nvme_ns_cmd.c b/lib/nvme/nvme_ns_cmd.c index 9d8b79989..cf7f215da 100644 --- a/lib/nvme/nvme_ns_cmd.c +++ b/lib/nvme/nvme_ns_cmd.c @@ -36,7 +36,8 @@ static struct nvme_request *_nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload, uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, - void *cb_arg, uint32_t opc, uint32_t io_flags); + void *cb_arg, uint32_t opc, uint32_t io_flags, + uint16_t apptag_mask, uint16_t apptag); static void nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl) @@ -98,19 +99,28 @@ _nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns, uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc, uint32_t io_flags, struct nvme_request *req, - uint32_t sectors_per_max_io, uint32_t sector_mask) + uint32_t sectors_per_max_io, uint32_t sector_mask, + uint16_t apptag_mask, uint16_t apptag) { uint32_t sector_size = ns->sector_size; + uint32_t md_size = ns->md_size; uint32_t remaining_lba_count = lba_count; uint32_t offset = 0; + uint32_t md_offset = 0; struct nvme_request *child, *tmp; + if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) { + /* for extended LBA only */ + if ((ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) && !(io_flags & SPDK_NVME_IO_FLAGS_PRACT)) + sector_size += ns->md_size; + } + while (remaining_lba_count > 0) { lba_count = sectors_per_max_io - (lba & sector_mask); lba_count = nvme_min(remaining_lba_count, lba_count); child = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, - cb_arg, opc, io_flags); + cb_arg, opc, io_flags, apptag_mask, apptag); if (child == NULL) { if (req->num_children) { /* free all child nvme_request */ @@ -123,10 +133,14 @@ _nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns, return NULL; } child->payload_offset = offset; + /* for separate metadata buffer only */ + if (payload->md) + child->md_offset = md_offset; nvme_request_add_child(req, child); remaining_lba_count -= lba_count; lba += lba_count; offset += lba_count * sector_size; + md_offset += lba_count * md_size; } return req; @@ -135,7 +149,7 @@ _nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns, static struct nvme_request * _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload, uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc, - uint32_t io_flags) + uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag) { struct nvme_request *req; struct spdk_nvme_cmd *cmd; @@ -153,6 +167,12 @@ _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload, sectors_per_max_io = ns->sectors_per_max_io; sectors_per_stripe = ns->sectors_per_stripe; + if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) { + /* for extended LBA only */ + if ((ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) && !(io_flags & SPDK_NVME_IO_FLAGS_PRACT)) + sector_size += ns->md_size; + } + req = nvme_allocate_request(payload, lba_count * sector_size, cb_fn, cb_arg); if (req == NULL) { return NULL; @@ -168,10 +188,10 @@ _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload, (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) { return _nvme_ns_cmd_split_request(ns, payload, lba, lba_count, cb_fn, cb_arg, opc, - io_flags, req, sectors_per_stripe, sectors_per_stripe - 1); + io_flags, req, sectors_per_stripe, sectors_per_stripe - 1, apptag_mask, apptag); } else if (lba_count > sectors_per_max_io) { return _nvme_ns_cmd_split_request(ns, payload, lba, lba_count, cb_fn, cb_arg, opc, - io_flags, req, sectors_per_max_io, 0); + io_flags, req, sectors_per_max_io, 0, apptag_mask, apptag); } else { cmd = &req->cmd; cmd->opc = opc; @@ -180,8 +200,20 @@ _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, const struct nvme_payload *payload, tmp_lba = (uint64_t *)&cmd->cdw10; *tmp_lba = lba; + if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) { + switch (ns->pi_type) { + case SPDK_NVME_FMT_NVM_PROTECTION_TYPE1: + case SPDK_NVME_FMT_NVM_PROTECTION_TYPE2: + cmd->cdw14 = (uint32_t)lba; + break; + } + } + cmd->cdw12 = lba_count - 1; cmd->cdw12 |= io_flags; + + cmd->cdw15 = apptag_mask; + cmd->cdw15 = (cmd->cdw15 << 16 | apptag); } return req; @@ -198,8 +230,33 @@ spdk_nvme_ns_cmd_read(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, vo payload.type = NVME_PAYLOAD_TYPE_CONTIG; payload.u.contig = buffer; + payload.md = NULL; - req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags); + req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags, 0, + 0); + if (req != NULL) { + return nvme_qpair_submit_request(qpair, req); + } else { + return ENOMEM; + } +} + +int +spdk_nvme_ns_cmd_read_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer, + void *metadata, + uint64_t lba, + uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, + uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag) +{ + struct nvme_request *req; + struct nvme_payload payload; + + payload.type = NVME_PAYLOAD_TYPE_CONTIG; + payload.u.contig = buffer; + payload.md = metadata; + + req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags, + apptag_mask, apptag); if (req != NULL) { return nvme_qpair_submit_request(qpair, req); } else { @@ -221,11 +278,13 @@ spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, return EINVAL; payload.type = NVME_PAYLOAD_TYPE_SGL; + payload.md = NULL; payload.u.sgl.reset_sgl_fn = reset_sgl_fn; payload.u.sgl.next_sge_fn = next_sge_fn; payload.u.sgl.cb_arg = cb_arg; - req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags); + req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, io_flags, 0, + 0); if (req != NULL) { return nvme_qpair_submit_request(qpair, req); } else { @@ -244,8 +303,32 @@ spdk_nvme_ns_cmd_write(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, payload.type = NVME_PAYLOAD_TYPE_CONTIG; payload.u.contig = buffer; + payload.md = NULL; - req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags); + req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags, 0, + 0); + if (req != NULL) { + return nvme_qpair_submit_request(qpair, req); + } else { + return ENOMEM; + } +} + +int +spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, + void *buffer, void *metadata, uint64_t lba, + uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, + uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag) +{ + struct nvme_request *req; + struct nvme_payload payload; + + payload.type = NVME_PAYLOAD_TYPE_CONTIG; + payload.u.contig = buffer; + payload.md = metadata; + + req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags, + apptag_mask, apptag); if (req != NULL) { return nvme_qpair_submit_request(qpair, req); } else { @@ -267,11 +350,13 @@ spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, return EINVAL; payload.type = NVME_PAYLOAD_TYPE_SGL; + payload.md = NULL; payload.u.sgl.reset_sgl_fn = reset_sgl_fn; payload.u.sgl.next_sge_fn = next_sge_fn; payload.u.sgl.cb_arg = cb_arg; - req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags); + req = _nvme_ns_cmd_rw(ns, &payload, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, io_flags, 0, + 0); if (req != NULL) { return nvme_qpair_submit_request(qpair, req); } else { diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c index 06a25133b..c09d55aaa 100644 --- a/lib/nvme/nvme_qpair.c +++ b/lib/nvme/nvme_qpair.c @@ -612,6 +612,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ uint64_t phys_addr; void *seg_addr; uint32_t nseg, cur_nseg, modulo, unaligned; + void *md_payload; void *payload = req->payload.u.contig + req->payload_offset; phys_addr = nvme_vtophys(payload); @@ -626,6 +627,15 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE)); } + if (req->payload.md) { + md_payload = req->payload.md + req->md_offset; + tr->req->cmd.mptr = nvme_vtophys(md_payload); + if (tr->req->cmd.mptr == NVME_VTOPHYS_ERROR) { + _nvme_fail_request_bad_vtophys(qpair, tr); + return -1; + } + } + tr->req->cmd.psdt = SPDK_NVME_PSDT_PRP; tr->req->cmd.dptr.prp.prp1 = phys_addr; if (nseg == 2) { diff --git a/test/lib/nvme/Makefile b/test/lib/nvme/Makefile index 507a7b51a..9ab4a2857 100644 --- a/test/lib/nvme/Makefile +++ b/test/lib/nvme/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(CURDIR)/../../.. include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -DIRS-y = unit aer reset sgl +DIRS-y = unit aer reset sgl e2edp .PHONY: all clean $(DIRS-y) diff --git a/test/lib/nvme/e2edp/.gitignore b/test/lib/nvme/e2edp/.gitignore new file mode 100644 index 000000000..df0958204 --- /dev/null +++ b/test/lib/nvme/e2edp/.gitignore @@ -0,0 +1 @@ +nvme_dp diff --git a/test/lib/nvme/e2edp/Makefile b/test/lib/nvme/e2edp/Makefile new file mode 100644 index 000000000..4a2c250c9 --- /dev/null +++ b/test/lib/nvme/e2edp/Makefile @@ -0,0 +1,57 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(CURDIR)/../../../.. +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +APP = nvme_dp + +C_SRCS := nvme_dp.c + +CFLAGS += -I. $(DPDK_INC) + +SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \ + $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \ + $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a + +LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) -lpthread $(DPDK_LIB) -lrt + +all : $(APP) + +$(APP) : $(OBJS) $(SPDK_LIBS) + $(LINK_C) + +clean : + $(CLEAN_C) $(APP) + +include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk diff --git a/test/lib/nvme/e2edp/nvme_dp.c b/test/lib/nvme/e2edp/nvme_dp.c new file mode 100644 index 000000000..e5f5fcc87 --- /dev/null +++ b/test/lib/nvme/e2edp/nvme_dp.c @@ -0,0 +1,626 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * NVMe end-to-end data protection test + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "spdk/nvme.h" +#include "spdk/pci.h" + +static uint32_t swap32(uint32_t value) +{ + uint32_t result = 0; + result |= (value & 0x000000FF) << 24; + result |= (value & 0x0000FF00) << 8; + result |= (value & 0x00FF0000) >> 8; + result |= (value & 0xFF000000) >> 24; + return result; +} + + +static uint16_t swap16(uint16_t value) +{ + uint16_t result = 0; + + result |= (value & 0x00FF) << 8; + result |= (value & 0xFF00) >> 8; + + return result; +} + +struct rte_mempool *request_mempool; + +#define MAX_DEVS 64 + +#define DATA_PATTERN 0x5A + +struct dev { + struct spdk_nvme_ctrlr *ctrlr; + char name[100]; +}; + +static struct dev devs[MAX_DEVS]; +static int num_devs = 0; + +#define foreach_dev(iter) \ + for (iter = devs; iter - devs < num_devs; iter++) + +static int io_complete_flag = 0; + +struct io_request { + void *contig; + void *metadata; + bool use_extended_lba; + uint64_t lba; + uint32_t lba_count; + uint16_t apptag_mask; + uint16_t apptag; +}; + +static void +io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) +{ + if (spdk_nvme_cpl_is_error(cpl)) + io_complete_flag = 2; + else + io_complete_flag = 1; +} + +/* + * No protection information with PRACT setting to 1, + * both extended LBA format and separate metadata can + * run the test case. + */ +static uint32_t dp_with_pract_test(struct spdk_nvme_ns *ns, struct io_request *req, + uint32_t *io_flags) +{ + uint32_t sector_size; + + req->lba_count = 8; + + sector_size = spdk_nvme_ns_get_sector_size(ns); + /* No additional metadata buffer provided */ + req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000); + if (!req->contig) + return 0; + + switch (spdk_nvme_ns_get_pi_type(ns)) { + case SPDK_NVME_FMT_NVM_PROTECTION_TYPE3: + *io_flags = SPDK_NVME_IO_FLAGS_PRCHK_GUARD | SPDK_NVME_IO_FLAGS_PRACT; + break; + case SPDK_NVME_FMT_NVM_PROTECTION_TYPE1: + case SPDK_NVME_FMT_NVM_PROTECTION_TYPE2: + *io_flags = SPDK_NVME_IO_FLAGS_PRCHK_GUARD | SPDK_NVME_IO_FLAGS_PRCHK_REFTAG | + SPDK_NVME_IO_FLAGS_PRACT; + break; + default: + *io_flags = 0; + break; + } + req->lba = 0x100000; + req->use_extended_lba = false; + req->metadata = NULL; + + return req->lba_count; +} + +/* Block Reference Tag checked for TYPE1 and TYPE2 with PRACT setting to 0 */ +static uint32_t dp_without_pract_extended_lba_test(struct spdk_nvme_ns *ns, struct io_request *req, + uint32_t *io_flags) +{ + struct spdk_nvme_protection_info *pi; + uint32_t md_size, sector_size; + + req->lba_count = 2; + + switch (spdk_nvme_ns_get_pi_type(ns)) { + case SPDK_NVME_FMT_NVM_PROTECTION_TYPE3: + return 0; + default: + break; + } + + /* extended LBA only for the test case */ + if (!(spdk_nvme_ns_supports_extended_lba(ns))) + return 0; + + sector_size = spdk_nvme_ns_get_sector_size(ns);; + md_size = spdk_nvme_ns_get_md_size(ns); + req->contig = rte_zmalloc(NULL, (sector_size + md_size) * req->lba_count, 0x1000); + if (!req->contig) + return 0; + + req->lba = 0x200000; + req->use_extended_lba = true; + req->metadata = NULL; + pi = (struct spdk_nvme_protection_info *)(req->contig + sector_size + md_size - 8); + /* big-endian for reference tag */ + pi->ref_tag = swap32((uint32_t)req->lba); + + pi = (struct spdk_nvme_protection_info *)(req->contig + (sector_size + md_size) * 2 - 8); + /* is incremented for each subsequent logical block */ + pi->ref_tag = swap32((uint32_t)req->lba + 1); + + *io_flags = SPDK_NVME_IO_FLAGS_PRCHK_REFTAG; + + return req->lba_count; +} + +/* LBA + Metadata without data protection bits setting */ +static uint32_t dp_without_flags_extended_lba_test(struct spdk_nvme_ns *ns, struct io_request *req, + uint32_t *io_flags) +{ + uint32_t md_size, sector_size; + + req->lba_count = 16; + + /* extended LBA only for the test case */ + if (!(spdk_nvme_ns_supports_extended_lba(ns))) + return 0; + + sector_size = spdk_nvme_ns_get_sector_size(ns);; + md_size = spdk_nvme_ns_get_md_size(ns); + req->contig = rte_zmalloc(NULL, (sector_size + md_size) * req->lba_count, 0x1000); + if (!req->contig) + return 0; + + req->lba = 0x400000; + req->use_extended_lba = true; + req->metadata = NULL; + *io_flags = 0; + + return req->lba_count; +} + +/* Block Reference Tag checked for TYPE1 and TYPE2 with PRACT setting to 0 */ +static uint32_t dp_without_pract_separate_meta_test(struct spdk_nvme_ns *ns, struct io_request *req, + uint32_t *io_flags) +{ + struct spdk_nvme_protection_info *pi; + uint32_t md_size, sector_size; + + req->lba_count = 2; + + switch (spdk_nvme_ns_get_pi_type(ns)) { + case SPDK_NVME_FMT_NVM_PROTECTION_TYPE3: + return 0; + default: + break; + } + + /* separate metadata payload for the test case */ + if (spdk_nvme_ns_supports_extended_lba(ns)) + return 0; + + sector_size = spdk_nvme_ns_get_sector_size(ns);; + md_size = spdk_nvme_ns_get_md_size(ns); + req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000); + if (!req->contig) + return 0; + + req->metadata = rte_zmalloc(NULL, md_size * req->lba_count, 0x1000); + if (!req->metadata) { + rte_free(req->contig); + return 0; + } + + req->lba = 0x400000; + req->use_extended_lba = false; + + /* last 8 bytes if the metadata size bigger than 8 */ + pi = (struct spdk_nvme_protection_info *)(req->metadata + md_size - 8); + /* big-endian for reference tag */ + pi->ref_tag = swap32((uint32_t)req->lba); + + pi = (struct spdk_nvme_protection_info *)(req->metadata + md_size * 2 - 8); + /* is incremented for each subsequent logical block */ + pi->ref_tag = swap32((uint32_t)req->lba + 1); + + *io_flags = SPDK_NVME_IO_FLAGS_PRCHK_REFTAG; + + return req->lba_count; +} + +/* Application Tag checked with PRACT setting to 0 */ +static uint32_t dp_without_pract_separate_meta_apptag_test(struct spdk_nvme_ns *ns, + struct io_request *req, + uint32_t *io_flags) +{ + struct spdk_nvme_protection_info *pi; + uint32_t md_size, sector_size; + + req->lba_count = 1; + + /* separate metadata payload for the test case */ + if (spdk_nvme_ns_supports_extended_lba(ns)) + return 0; + + sector_size = spdk_nvme_ns_get_sector_size(ns);; + md_size = spdk_nvme_ns_get_md_size(ns); + req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000); + if (!req->contig) + return 0; + + req->metadata = rte_zmalloc(NULL, md_size * req->lba_count, 0x1000); + if (!req->metadata) { + rte_free(req->contig); + return 0; + } + + req->lba = 0x500000; + req->use_extended_lba = false; + req->apptag_mask = 0xFFFF; + req->apptag = req->lba_count; + + /* last 8 bytes if the metadata size bigger than 8 */ + pi = (struct spdk_nvme_protection_info *)(req->metadata + md_size - 8); + pi->app_tag = swap16(req->lba_count); + + *io_flags = SPDK_NVME_IO_FLAGS_PRCHK_APPTAG; + + return req->lba_count; +} + +/* + * LBA + Metadata without data protection bits setting, + * separate metadata payload for the test case. + */ +static uint32_t dp_without_flags_separate_meta_test(struct spdk_nvme_ns *ns, struct io_request *req, + uint32_t *io_flags) +{ + uint32_t md_size, sector_size; + + req->lba_count = 16; + + /* separate metadata payload for the test case */ + if (spdk_nvme_ns_supports_extended_lba(ns)) + return 0; + + sector_size = spdk_nvme_ns_get_sector_size(ns);; + md_size = spdk_nvme_ns_get_md_size(ns); + req->contig = rte_zmalloc(NULL, sector_size * req->lba_count, 0x1000); + if (!req->contig) + return 0; + + req->metadata = rte_zmalloc(NULL, md_size * req->lba_count, 0x1000); + if (!req->metadata) { + rte_free(req->contig); + return 0; + } + + req->lba = 0x600000; + req->use_extended_lba = false; + *io_flags = 0; + + return req->lba_count; +} + +typedef uint32_t (*nvme_build_io_req_fn_t)(struct spdk_nvme_ns *ns, struct io_request *req, + uint32_t *lba_count); + +static void +free_req(struct io_request *req) +{ + if (req == NULL) { + return; + } + + if (req->contig) + rte_free(req->contig); + + if (req->metadata) + rte_free(req->metadata); + + rte_free(req); +} + +static void +ns_data_buffer_reset(struct spdk_nvme_ns *ns, struct io_request *req, uint8_t data_pattern) +{ + uint32_t md_size, sector_size; + uint32_t i, offset = 0; + uint8_t *buf; + + sector_size = spdk_nvme_ns_get_sector_size(ns); + md_size = spdk_nvme_ns_get_md_size(ns); + + for (i = 0; i < req->lba_count; i++) { + if (req->use_extended_lba) + offset = (sector_size + md_size) * i; + else + offset = sector_size * i; + + buf = (uint8_t *)req->contig + offset; + memset(buf, data_pattern, sector_size); + } +} + +static int +ns_data_buffer_compare(struct spdk_nvme_ns *ns, struct io_request *req, uint8_t data_pattern) +{ + uint32_t md_size, sector_size; + uint32_t i, j, offset = 0; + uint8_t *buf; + + sector_size = spdk_nvme_ns_get_sector_size(ns); + md_size = spdk_nvme_ns_get_md_size(ns); + + for (i = 0; i < req->lba_count; i++) { + if (req->use_extended_lba) + offset = (sector_size + md_size) * i; + else + offset = sector_size * i; + + buf = (uint8_t *)req->contig + offset; + for (j = 0; j < sector_size; j++) { + if (buf[j] != data_pattern) { + return -1; + } + } + } + + return 0; +} + +static int +write_read_e2e_dp_tests(struct dev *dev, nvme_build_io_req_fn_t build_io_fn, const char *test_name) +{ + int rc = 0; + uint32_t lba_count; + uint32_t io_flags = 0; + + struct io_request *req; + struct spdk_nvme_ns *ns; + struct spdk_nvme_qpair *qpair; + const struct spdk_nvme_ns_data *nsdata; + + ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, 1); + if (!ns) { + fprintf(stderr, "Null namespace\n"); + return 0; + } + + if (!(spdk_nvme_ns_get_flags(ns) & SPDK_NVME_NS_DPS_PI_SUPPORTED)) + return 0; + + nsdata = spdk_nvme_ns_get_data(ns); + if (!nsdata || !spdk_nvme_ns_get_sector_size(ns)) { + fprintf(stderr, "Empty nsdata or wrong sector size\n"); + return 0; + } + + req = rte_zmalloc(NULL, sizeof(*req), 0); + if (!req) { + fprintf(stderr, "Allocate request failed\n"); + return 0; + } + + /* IO parameters setting */ + lba_count = build_io_fn(ns, req, &io_flags); + + if (!lba_count) { + fprintf(stderr, "%s: %s bypass the test case\n", dev->name, test_name); + free_req(req); + return 0; + } + + qpair = spdk_nvme_ctrlr_alloc_io_qpair(dev->ctrlr, 0); + if (!qpair) { + free_req(req); + return -1; + } + + ns_data_buffer_reset(ns, req, DATA_PATTERN); + if (req->use_extended_lba) + rc = spdk_nvme_ns_cmd_write(ns, qpair, req->contig, req->lba, lba_count, + io_complete, req, io_flags); + else + rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, req->contig, req->metadata, req->lba, lba_count, + io_complete, req, io_flags, req->apptag_mask, req->apptag); + + if (rc != 0) { + fprintf(stderr, "%s: %s write submit failed\n", dev->name, test_name); + spdk_nvme_ctrlr_free_io_qpair(qpair); + free_req(req); + return -1; + } + + io_complete_flag = 0; + + while (!io_complete_flag) + spdk_nvme_qpair_process_completions(qpair, 1); + + if (io_complete_flag != 1) { + fprintf(stderr, "%s: %s write exec failed\n", dev->name, test_name); + spdk_nvme_ctrlr_free_io_qpair(qpair); + free_req(req); + return -1; + } + + /* reset completion flag */ + io_complete_flag = 0; + + ns_data_buffer_reset(ns, req, 0); + if (req->use_extended_lba) + rc = spdk_nvme_ns_cmd_read(ns, qpair, req->contig, req->lba, lba_count, + io_complete, req, io_flags); + else + rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, req->contig, req->metadata, req->lba, lba_count, + io_complete, req, io_flags, req->apptag_mask, req->apptag); + + if (rc != 0) { + fprintf(stderr, "%s: %s read failed\n", dev->name, test_name); + spdk_nvme_ctrlr_free_io_qpair(qpair); + free_req(req); + return -1; + } + + while (!io_complete_flag) + spdk_nvme_qpair_process_completions(qpair, 1); + + if (io_complete_flag != 1) { + fprintf(stderr, "%s: %s read failed\n", dev->name, test_name); + spdk_nvme_ctrlr_free_io_qpair(qpair); + free_req(req); + return -1; + } + + rc = ns_data_buffer_compare(ns, req, DATA_PATTERN); + if (rc < 0) { + fprintf(stderr, "%s: %s write/read success, but memcmp Failed\n", dev->name, test_name); + spdk_nvme_ctrlr_free_io_qpair(qpair); + free_req(req); + return -1; + } + + fprintf(stdout, "%s: %s test passed\n", dev->name, test_name); + spdk_nvme_ctrlr_free_io_qpair(qpair); + free_req(req); + return rc; +} + +static bool +probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts) +{ + if (spdk_pci_device_has_non_uio_driver(dev)) { + fprintf(stderr, "non-uio kernel driver attached to NVMe\n"); + fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n", + spdk_pci_device_get_domain(dev), + spdk_pci_device_get_bus(dev), + spdk_pci_device_get_dev(dev), + spdk_pci_device_get_func(dev)); + fprintf(stderr, " skipping...\n"); + return false; + } + + printf("Attaching to %04x:%02x:%02x.%02x\n", + spdk_pci_device_get_domain(dev), + spdk_pci_device_get_bus(dev), + spdk_pci_device_get_dev(dev), + spdk_pci_device_get_func(dev)); + + return true; +} + +static void +attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_nvme_ctrlr *ctrlr, + const struct spdk_nvme_ctrlr_opts *opts) +{ + struct dev *dev; + + /* add to dev list */ + dev = &devs[num_devs++]; + + dev->ctrlr = ctrlr; + + snprintf(dev->name, sizeof(dev->name), "%04X:%02X:%02X.%02X", + spdk_pci_device_get_domain(pci_dev), + spdk_pci_device_get_bus(pci_dev), + spdk_pci_device_get_dev(pci_dev), + spdk_pci_device_get_func(pci_dev)); + + printf("Attached to %s\n", dev->name); +} + + +static const char *ealargs[] = { + "nvme_dp", + "-c 0x1", + "-n 4", +}; + +int main(int argc, char **argv) +{ + struct dev *iter; + int rc, i; + + printf("NVMe Write/Read with End-to-End data protection test\n"); + + rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), + (char **)(void *)(uintptr_t)ealargs); + + if (rc < 0) { + fprintf(stderr, "could not initialize dpdk\n"); + exit(1); + } + + request_mempool = rte_mempool_create("nvme_request", 8192, + spdk_nvme_request_size(), 128, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + + if (request_mempool == NULL) { + fprintf(stderr, "could not initialize request mempool\n"); + exit(1); + } + + if (spdk_nvme_probe(NULL, probe_cb, attach_cb) != 0) { + fprintf(stderr, "nvme_probe() failed\n"); + exit(1); + } + + rc = 0; + foreach_dev(iter) { +#define TEST(x) write_read_e2e_dp_tests(iter, x, #x) + if (TEST(dp_with_pract_test) + || TEST(dp_without_pract_extended_lba_test) + || TEST(dp_without_flags_extended_lba_test) + || TEST(dp_without_pract_separate_meta_test) + || TEST(dp_without_pract_separate_meta_apptag_test) + || TEST(dp_without_flags_separate_meta_test)) { +#undef TEST + rc = 1; + printf("%s: failed End-to-End data protection tests\n", iter->name); + } + } + + printf("Cleaning up...\n"); + + for (i = 0; i < num_devs; i++) { + struct dev *dev = &devs[i]; + + spdk_nvme_detach(dev->ctrlr); + } + + return rc; +} diff --git a/test/lib/nvme/nvme.sh b/test/lib/nvme/nvme.sh index 49577cb22..55db9adfb 100755 --- a/test/lib/nvme/nvme.sh +++ b/test/lib/nvme/nvme.sh @@ -43,4 +43,8 @@ timing_enter sgl $testdir/sgl/sgl timing_exit sgl +timing_enter e2edp +$testdir/e2edp/nvme_dp +timing_exit e2edp + timing_exit nvme