diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc86ab70..5ded1209a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## v18.07: (Upcoming Release) +### NVMe Driver + +New API function spdk_nvme_qpair_add_cmd_error_injection() and +spdk_nvme_qpair_remove_cmd_error_injection() have been added for NVMe error emulation, +users can set specified command with specified error status for error emulation. + ### Build System The build system now generates a combined shared library (libspdk.so) that may be used diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index c9acabb65..6c7dd0826 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -1679,6 +1679,54 @@ int spdk_nvme_ns_cmd_compare_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_q void *cb_arg, uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag); +/** + * \brief Inject an error for the next request with a given opcode. + * + * \param ctrlr NVMe controller. + * \param qpair I/O queue pair to add the error command, + * NULL for Admin queue pair. + * \param opc Opcode for Admin or I/O commands. + * \param do_not_submit True if matching requests should not be submitted + * to the controller, but instead completed manually + * after timeout_in_us has expired. False if matching + * requests should be submitted to the controller and + * have their completion status modified after the + * controller completes the request. + * \param timeout_in_us Wait specified microseconds when do_not_submit is true. + * \param err_count Number of matching requests to inject errors. + * \param sct Status code type. + * \param sc Status code. + * + * \return 0 if successfully enabled, ENOMEM if an error command + * structure cannot be allocated. + * + * The function can be called multiple times to inject errors for different + * commands. If the opcode matches an existing entry, the existing entry + * will be updated with the values specified. + */ +int spdk_nvme_qpair_add_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr, + struct spdk_nvme_qpair *qpair, + uint8_t opc, + bool do_not_submit, + uint64_t timeout_in_us, + uint32_t err_count, + uint8_t sct, uint8_t sc); + +/** + * \brief Clear the specified NVMe command with error status. + * + * \param ctrlr NVMe controller. + * \param qpair I/O queue pair to remove the error command, + * \ NULL for Admin queue pair. + * \param opc Opcode for Admin or I/O commands. + * + * The function will remove specified command in the error list. + */ +void spdk_nvme_qpair_remove_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr, + struct spdk_nvme_qpair *qpair, + uint8_t opc); + + #ifdef __cplusplus } #endif diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index fc9dbf49e..3bf9d9289 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -34,6 +34,7 @@ #ifndef __NVME_INTERNAL_H__ #define __NVME_INTERNAL_H__ +#include "spdk/likely.h" #include "spdk/stdinc.h" #include "spdk/nvme.h" @@ -186,6 +187,15 @@ nvme_payload_type(const struct nvme_payload *payload) { return payload->reset_sgl_fn ? NVME_PAYLOAD_TYPE_SGL : NVME_PAYLOAD_TYPE_CONTIG; } +struct nvme_error_cmd { + bool do_not_submit; + uint64_t timeout_tsc; + uint32_t err_count; + uint8_t opc; + struct spdk_nvme_status status; + TAILQ_ENTRY(nvme_error_cmd) link; +}; + struct nvme_request { struct spdk_nvme_cmd cmd; @@ -208,6 +218,12 @@ struct nvme_request { uint32_t payload_size; + /** + * Timeout ticks for error injection requests, can be extended in future + * to support per-request timeout feature. + */ + uint64_t timeout_tsc; + /** * Data payload for this request's command. */ @@ -293,6 +309,10 @@ struct nvme_async_event_request { struct spdk_nvme_qpair { STAILQ_HEAD(, nvme_request) free_req; STAILQ_HEAD(, nvme_request) queued_req; + /** Commands opcode in this list will return error */ + TAILQ_HEAD(, nvme_error_cmd) err_cmd_head; + /** Requests in this list will return error */ + STAILQ_HEAD(, nvme_request) err_req_head; enum spdk_nvme_transport_type trtype; @@ -722,6 +742,33 @@ struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpa static inline void nvme_complete_request(struct nvme_request *req, struct spdk_nvme_cpl *cpl) { + struct spdk_nvme_qpair *qpair = req->qpair; + struct spdk_nvme_cpl err_cpl; + struct nvme_error_cmd *cmd; + + /* error injection at completion path, + * only inject for successful completed commands + */ + if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head) && + !spdk_nvme_cpl_is_error(cpl))) { + TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) { + + if (cmd->do_not_submit) { + continue; + } + + if ((cmd->opc == req->cmd.opc) && cmd->err_count) { + + err_cpl = *cpl; + err_cpl.status.sct = cmd->status.sct; + err_cpl.status.sc = cmd->status.sc; + + cpl = &err_cpl; + cmd->err_count--; + } + } + } + if (req->cb_fn) { req->cb_fn(req->cb_arg, cpl); } diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c index b738ed2f4..62bc7b92f 100644 --- a/lib/nvme/nvme_qpair.c +++ b/lib/nvme/nvme_qpair.c @@ -377,12 +377,25 @@ int32_t spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions) { int32_t ret; + struct nvme_request *req, *tmp; if (qpair->ctrlr->is_failed) { nvme_qpair_fail(qpair); return 0; } + /* error injection for those queued error requests */ + if (spdk_unlikely(!STAILQ_EMPTY(&qpair->err_req_head))) { + STAILQ_FOREACH_SAFE(req, &qpair->err_req_head, stailq, tmp) { + if (spdk_get_ticks() - req->submit_tick > req->timeout_tsc) { + STAILQ_REMOVE(&qpair->err_req_head, req, nvme_request, stailq); + nvme_qpair_manual_complete_request(qpair, req, + req->cpl.status.sct, + req->cpl.status.sc, true); + } + } + } + qpair->in_completion_context = 1; ret = nvme_transport_qpair_process_completions(qpair, max_completions); qpair->in_completion_context = 0; @@ -417,6 +430,8 @@ nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, STAILQ_INIT(&qpair->free_req); STAILQ_INIT(&qpair->queued_req); + TAILQ_INIT(&qpair->err_cmd_head); + STAILQ_INIT(&qpair->err_req_head); req_size_padded = (sizeof(struct nvme_request) + 63) & ~(size_t)63; @@ -437,6 +452,22 @@ nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, void nvme_qpair_deinit(struct spdk_nvme_qpair *qpair) { + struct nvme_request *req; + struct nvme_error_cmd *cmd, *entry; + + while (!STAILQ_EMPTY(&qpair->err_req_head)) { + req = STAILQ_FIRST(&qpair->err_req_head); + STAILQ_REMOVE_HEAD(&qpair->err_req_head, stailq); + nvme_qpair_manual_complete_request(qpair, req, + req->cpl.status.sct, + req->cpl.status.sc, true); + } + + TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) { + TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link); + spdk_dma_free(cmd); + } + spdk_dma_free(qpair->req_buf); } @@ -445,6 +476,7 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re { int rc = 0; struct nvme_request *child_req, *tmp; + struct nvme_error_cmd *cmd; struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; bool child_req_failed = false; @@ -473,6 +505,26 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re return rc; } + /* queue those requests which matches with opcode in err_cmd list */ + if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head))) { + TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) { + if (!cmd->do_not_submit) { + continue; + } + + if ((cmd->opc == req->cmd.opc) && cmd->err_count) { + /* add to error request list and set cpl */ + req->timeout_tsc = cmd->timeout_tsc; + req->submit_tick = spdk_get_ticks(); + req->cpl.status.sct = cmd->status.sct; + req->cpl.status.sc = cmd->status.sc; + STAILQ_INSERT_TAIL(&qpair->err_req_head, req, stailq); + cmd->err_count--; + return 0; + } + } + } + return nvme_transport_qpair_submit_request(qpair, req); } @@ -504,6 +556,16 @@ nvme_qpair_enable(struct spdk_nvme_qpair *qpair) void nvme_qpair_disable(struct spdk_nvme_qpair *qpair) { + struct nvme_request *req; + + while (!STAILQ_EMPTY(&qpair->err_req_head)) { + req = STAILQ_FIRST(&qpair->err_req_head); + STAILQ_REMOVE_HEAD(&qpair->err_req_head, stailq); + nvme_qpair_manual_complete_request(qpair, req, + req->cpl.status.sct, + req->cpl.status.sc, true); + } + nvme_transport_qpair_disable(qpair); } @@ -522,3 +584,64 @@ nvme_qpair_fail(struct spdk_nvme_qpair *qpair) nvme_transport_qpair_fail(qpair); } + +int +spdk_nvme_qpair_add_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr, + struct spdk_nvme_qpair *qpair, + uint8_t opc, bool do_not_submit, + uint64_t timeout_in_us, + uint32_t err_count, + uint8_t sct, uint8_t sc) +{ + struct nvme_error_cmd *entry, *cmd = NULL; + + if (qpair == NULL) { + qpair = ctrlr->adminq; + } + + TAILQ_FOREACH(entry, &qpair->err_cmd_head, link) { + if (entry->opc == opc) { + cmd = entry; + break; + } + } + + if (cmd == NULL) { + cmd = spdk_dma_zmalloc(sizeof(*cmd), 64, NULL); + if (!cmd) { + return -ENOMEM; + } + TAILQ_INSERT_TAIL(&qpair->err_cmd_head, cmd, link); + } + + cmd->do_not_submit = do_not_submit; + cmd->err_count = err_count; + cmd->timeout_tsc = timeout_in_us * spdk_get_ticks_hz() / 1000000ULL; + cmd->opc = opc; + cmd->status.sct = sct; + cmd->status.sc = sc; + + return 0; +} + +void +spdk_nvme_qpair_remove_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr, + struct spdk_nvme_qpair *qpair, + uint8_t opc) +{ + struct nvme_error_cmd *cmd, *entry; + + if (qpair == NULL) { + qpair = ctrlr->adminq; + } + + TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) { + if (cmd->opc == opc) { + TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link); + spdk_dma_free(cmd); + return; + } + } + + return; +} diff --git a/test/nvme/Makefile b/test/nvme/Makefile index 2a8236682..9fae60f6e 100644 --- a/test/nvme/Makefile +++ b/test/nvme/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -DIRS-y = aer reset sgl e2edp overhead deallocated_value +DIRS-y = aer reset sgl e2edp overhead deallocated_value err_injection .PHONY: all clean $(DIRS-y) diff --git a/test/nvme/err_injection/.gitignore b/test/nvme/err_injection/.gitignore new file mode 100644 index 000000000..3572a8e78 --- /dev/null +++ b/test/nvme/err_injection/.gitignore @@ -0,0 +1 @@ +err_injection diff --git a/test/nvme/err_injection/Makefile b/test/nvme/err_injection/Makefile new file mode 100644 index 000000000..4f5f18514 --- /dev/null +++ b/test/nvme/err_injection/Makefile @@ -0,0 +1,39 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +APP = err_injection + +include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk diff --git a/test/nvme/err_injection/err_injection.c b/test/nvme/err_injection/err_injection.c new file mode 100644 index 000000000..942e31ed1 --- /dev/null +++ b/test/nvme/err_injection/err_injection.c @@ -0,0 +1,273 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "spdk/log.h" +#include "spdk/nvme.h" +#include "spdk/env.h" + +#define MAX_DEVS 64 + +struct dev { + bool error_expected; + struct spdk_nvme_ctrlr *ctrlr; + struct spdk_nvme_ns *ns; + struct spdk_nvme_qpair *qpair; + void *data; + char name[SPDK_NVMF_TRADDR_MAX_LEN + 1]; +}; + +static struct dev devs[MAX_DEVS]; +static int num_devs = 0; + +#define foreach_dev(iter) \ + for (iter = devs; iter - devs < num_devs; iter++) + +static int outstanding_commands = 0; +static int failed = 0; + +static bool +probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr_opts *opts) +{ + printf("Attaching to %s\n", trid->traddr); + + return true; +} + +static void +attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) +{ + struct dev *dev; + uint32_t nsid; + + /* add to dev list */ + dev = &devs[num_devs++]; + if (num_devs >= MAX_DEVS) { + return; + } + + dev->ctrlr = ctrlr; + nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); + dev->ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); + if (dev->ns == NULL) { + failed = 1; + return; + } + dev->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0); + if (dev->qpair == NULL) { + failed = 1; + return; + } + + snprintf(dev->name, sizeof(dev->name), "%s", + trid->traddr); + + printf("Attached to %s\n", dev->name); +} + +static void +get_feature_test_cb(void *cb_arg, const struct spdk_nvme_cpl *cpl) +{ + struct dev *dev = cb_arg; + + outstanding_commands--; + + if (spdk_nvme_cpl_is_error(cpl) && dev->error_expected) { + if (cpl->status.sct != SPDK_NVME_SCT_GENERIC || + cpl->status.sc != SPDK_NVME_SC_INVALID_FIELD) { + failed = 1; + } + printf("%s: get features failed as expected\n", dev->name); + return; + } + + if (!spdk_nvme_cpl_is_error(cpl) && !dev->error_expected) { + printf("%s: get features successfully as expected\n", dev->name); + return; + } + + failed = 1; +} + +static void +get_feature_test(bool error_expected) +{ + struct dev *dev; + struct spdk_nvme_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.opc = SPDK_NVME_OPC_GET_FEATURES; + cmd.cdw10 = SPDK_NVME_FEAT_NUMBER_OF_QUEUES; + + foreach_dev(dev) { + dev->error_expected = error_expected; + if (spdk_nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0, + get_feature_test_cb, dev) != 0) { + printf("Error: failed to send Get Features command for dev=%p\n", dev); + failed = 1; + goto cleanup; + } + outstanding_commands++; + } + +cleanup: + + while (outstanding_commands) { + foreach_dev(dev) { + spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); + } + } +} + +static void +read_test_cb(void *cb_arg, const struct spdk_nvme_cpl *cpl) +{ + struct dev *dev = cb_arg; + + outstanding_commands--; + spdk_dma_free(dev->data); + + if (spdk_nvme_cpl_is_error(cpl) && dev->error_expected) { + if (cpl->status.sct != SPDK_NVME_SCT_MEDIA_ERROR || + cpl->status.sc != SPDK_NVME_SC_UNRECOVERED_READ_ERROR) { + failed = 1; + } + printf("%s: read failed as expected\n", dev->name); + return; + } + + if (!spdk_nvme_cpl_is_error(cpl) && !dev->error_expected) { + printf("%s: read successfully as expected\n", dev->name); + return; + } + + failed = 1; +} + +static void +read_test(bool error_expected) +{ + struct dev *dev; + + foreach_dev(dev) { + dev->error_expected = error_expected; + dev->data = spdk_dma_zmalloc(0x1000, 0x1000, NULL); + if (!dev->data) { + failed = 1; + goto cleanup; + } + + if (spdk_nvme_ns_cmd_read(dev->ns, dev->qpair, dev->data, + 0, 1, read_test_cb, dev, 0) != 0) { + printf("Error: failed to send Read command for dev=%p\n", dev); + failed = 1; + goto cleanup; + } + + outstanding_commands++; + } + +cleanup: + + while (outstanding_commands) { + foreach_dev(dev) { + spdk_nvme_qpair_process_completions(dev->qpair, 0); + } + } +} + +int main(int argc, char **argv) +{ + struct dev *dev; + int i; + struct spdk_env_opts opts; + int rc; + + spdk_env_opts_init(&opts); + opts.name = "err_injection"; + opts.core_mask = "0x1"; + opts.mem_size = 64; + if (spdk_env_init(&opts) < 0) { + fprintf(stderr, "Unable to initialize SPDK env\n"); + return 1; + } + + printf("NVMe Error Injection test\n"); + + if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) { + fprintf(stderr, "spdk_nvme_probe() failed\n"); + return 1; + } + + if (failed) { + goto exit; + } + + foreach_dev(dev) { + /* Admin error injection at submission path */ + rc = spdk_nvme_qpair_add_cmd_error_injection(dev->ctrlr, NULL, + SPDK_NVME_OPC_GET_FEATURES, true, 5000, 1, + SPDK_NVME_SCT_GENERIC, SPDK_NVME_SC_INVALID_FIELD); + failed += rc; + /* IO error injection at completion path */ + rc = spdk_nvme_qpair_add_cmd_error_injection(dev->ctrlr, dev->qpair, + SPDK_NVME_OPC_READ, false, 0, 1, + SPDK_NVME_SCT_MEDIA_ERROR, SPDK_NVME_SC_UNRECOVERED_READ_ERROR); + failed += rc; + } + + if (failed) { + goto exit; + } + + /* Admin Get Feature, expect error return */ + get_feature_test(true); + /* Admin Get Feature, expect successful return */ + get_feature_test(false); + /* Read, expect error return */ + read_test(true); + /* Read, expect successful return */ + read_test(false); + +exit: + printf("Cleaning up...\n"); + for (i = 0; i < num_devs; i++) { + struct dev *dev = &devs[i]; + spdk_nvme_detach(dev->ctrlr); + } + + return failed; +} diff --git a/test/nvme/nvme.sh b/test/nvme/nvme.sh index 59d2a261f..18b8800f5 100755 --- a/test/nvme/nvme.sh +++ b/test/nvme/nvme.sh @@ -139,6 +139,10 @@ timing_enter e2edp $testdir/e2edp/nvme_dp timing_exit e2edp +timing_enter err_injection +$testdir/err_injection/err_injection +timing_exit err_injection + timing_enter overhead $testdir/overhead/overhead -s 4096 -t 1 -H timing_exit overhead