diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c index 8e0f9149c..61fa8dfde 100644 --- a/lib/nvme/nvme_qpair.c +++ b/lib/nvme/nvme_qpair.c @@ -798,6 +798,7 @@ _nvme_io_qpair_enable(struct nvme_qpair *qpair) struct nvme_tracker *tr_temp; struct nvme_request *req; + qpair->is_enabled = true; /* * Manually abort each outstanding I/O. This normally results in a * retry, unless the retry count on the associated request has @@ -809,7 +810,6 @@ _nvme_io_qpair_enable(struct nvme_qpair *qpair) NVME_SC_ABORTED_BY_REQUEST, 0, true); } - qpair->is_enabled = true; STAILQ_INIT(&temp); STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request); diff --git a/test/lib/nvme/Makefile b/test/lib/nvme/Makefile index 44a663177..c0bcd4f34 100644 --- a/test/lib/nvme/Makefile +++ b/test/lib/nvme/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(CURDIR)/../../.. include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -DIRS-y = unit aer +DIRS-y = unit aer reset .PHONY: all clean $(DIRS-y) diff --git a/test/lib/nvme/nvme.sh b/test/lib/nvme/nvme.sh index efd4e947a..344b74c6a 100755 --- a/test/lib/nvme/nvme.sh +++ b/test/lib/nvme/nvme.sh @@ -29,4 +29,11 @@ $rootdir/examples/nvme/perf/perf -q 128 -w read -s 12288 -t 5 process_core timing_exit perf +#Now test nvme reset function +timing_enter reset +$testdir/reset/reset -q 64 -w write -s 4096 -t 15 +process_core +timing_exit reset + + timing_exit nvme diff --git a/test/lib/nvme/reset/Makefile b/test/lib/nvme/reset/Makefile new file mode 100644 index 000000000..05d8e80c7 --- /dev/null +++ b/test/lib/nvme/reset/Makefile @@ -0,0 +1,63 @@ +# +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(CURDIR)/../../../.. +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +APP = reset + +C_SRCS := reset.c + +CFLAGS += -I. $(DPDK_INC) + +SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \ + $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \ + $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a + +LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt + + +OBJS = $(C_SRCS:.c=.o) + +all : $(APP) + +objs : $(OBJS) + +$(APP) : $(OBJS) $(SPDK_LIBS) + @echo " LINK $@" + $(Q)$(CC) $(CPPFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) + +clean : + $(Q)rm -f $(OBJS) *.d $(APP) + +include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk diff --git a/test/lib/nvme/reset/reset.c b/test/lib/nvme/reset/reset.c new file mode 100644 index 000000000..d35acb4ba --- /dev/null +++ b/test/lib/nvme/reset/reset.c @@ -0,0 +1,710 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#include "spdk/file.h" +#include "spdk/nvme.h" +#include "spdk/pci.h" +#include "spdk/string.h" + +struct ctrlr_entry { + struct nvme_controller *ctrlr; + struct ctrlr_entry *next; + char name[1024]; +}; + +struct ns_entry { + struct nvme_namespace *ns; + struct ns_entry *next; + uint32_t io_size_blocks; + uint64_t size_in_ios; + char name[1024]; +}; + +struct ns_worker_ctx { + struct ns_entry *entry; + struct ctrlr_entry *ctr_entry; + uint64_t io_completed; + uint64_t io_completed_error; + uint64_t io_submitted; + uint64_t current_queue_depth; + uint64_t offset_in_ios; + bool is_draining; + + struct ns_worker_ctx *next; +}; + +struct reset_task { + struct ns_worker_ctx *ns_ctx; + void *buf; +}; + +struct worker_thread { + struct ns_worker_ctx *ns_ctx; + unsigned lcore; +}; + +struct rte_mempool *request_mempool; +static struct rte_mempool *task_pool; + +static struct ctrlr_entry *g_controllers = NULL; +static struct ns_entry *g_namespaces = NULL; +static int g_num_namespaces = 0; +static struct worker_thread *g_workers = NULL; +static int g_num_workers = 0; +uint32_t g_max_completions = 0; + +static uint64_t g_tsc_rate; + +static int g_io_size_bytes; +static int g_rw_percentage; +static int g_is_random; +static int g_queue_depth; +static int g_time_in_sec; + +static void +task_complete(struct reset_task *task, const struct nvme_completion *completion); + +static void +register_ns(struct nvme_controller *ctrlr, struct pci_device *pci_dev, struct nvme_namespace *ns) +{ + struct ns_entry *entry; + const struct nvme_controller_data *cdata; + + entry = malloc(sizeof(struct ns_entry)); + if (entry == NULL) { + perror("ns_entry malloc"); + exit(1); + } + + cdata = nvme_ctrlr_get_data(ctrlr); + + entry->ns = ns; + entry->size_in_ios = nvme_ns_get_size(ns) / + g_io_size_bytes; + entry->io_size_blocks = g_io_size_bytes / nvme_ns_get_sector_size(ns); + + snprintf(entry->name, 44, "%-20.20s (%-20.20s)", cdata->mn, cdata->sn); + + g_num_namespaces++; + entry->next = g_namespaces; + g_namespaces = entry; +} + +static void +register_ctrlr(struct nvme_controller *ctrlr, struct pci_device *pci_dev) +{ + int nsid, num_ns; + struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry)); + + if (entry == NULL) { + perror("ctrlr_entry malloc"); + exit(1); + } + + entry->ctrlr = ctrlr; + entry->next = g_controllers; + g_controllers = entry; + + num_ns = nvme_ctrlr_get_num_ns(ctrlr); + for (nsid = 1; nsid <= num_ns; nsid++) { + register_ns(ctrlr, pci_dev, nvme_ctrlr_get_ns(ctrlr, nsid)); + } + +} + +static void task_ctor(struct rte_mempool *mp, void *arg, void *__task, unsigned id) +{ + struct reset_task *task = __task; + task->buf = rte_malloc(NULL, g_io_size_bytes, 0x200); + if (task->buf == NULL) { + fprintf(stderr, "task->buf rte_malloc failed\n"); + exit(1); + } +} + +static void io_complete(void *ctx, const struct nvme_completion *completion); + +static __thread unsigned int seed = 0; + +static void +submit_single_io(struct ns_worker_ctx *ns_ctx) +{ + struct reset_task *task = NULL; + uint64_t offset_in_ios; + int rc; + struct ns_entry *entry = ns_ctx->entry; + + if (rte_mempool_get(task_pool, (void **)&task) != 0) { + fprintf(stderr, "task_pool rte_mempool_get failed\n"); + exit(1); + } + + task->ns_ctx = ns_ctx; + task->ns_ctx->io_submitted++; + + if (g_is_random) { + offset_in_ios = rand_r(&seed) % entry->size_in_ios; + } else { + offset_in_ios = ns_ctx->offset_in_ios++; + if (ns_ctx->offset_in_ios == entry->size_in_ios) { + ns_ctx->offset_in_ios = 0; + } + } + + if ((g_rw_percentage == 100) || + (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) { + rc = nvme_ns_cmd_read(entry->ns, task->buf, offset_in_ios * entry->io_size_blocks, + entry->io_size_blocks, io_complete, task); + } else { + rc = nvme_ns_cmd_write(entry->ns, task->buf, offset_in_ios * entry->io_size_blocks, + entry->io_size_blocks, io_complete, task); + } + + if (rc != 0) { + fprintf(stderr, "starting I/O failed\n"); + } + + ns_ctx->current_queue_depth++; +} + +static void +task_complete(struct reset_task *task, const struct nvme_completion *completion) +{ + struct ns_worker_ctx *ns_ctx; + + ns_ctx = task->ns_ctx; + ns_ctx->current_queue_depth--; + + if (nvme_completion_is_error(completion)) { + ns_ctx->io_completed_error++; + } else { + ns_ctx->io_completed++; + } + + rte_mempool_put(task_pool, task); + + /* + * is_draining indicates when time has expired for the test run + * and we are just waiting for the previously submitted I/O + * to complete. In this case, do not submit a new I/O to replace + * the one just completed. + */ + if (!ns_ctx->is_draining) { + submit_single_io(ns_ctx); + } +} + +static void +io_complete(void *ctx, const struct nvme_completion *completion) +{ + task_complete((struct reset_task *)ctx, completion); +} + +static void +check_io(struct ns_worker_ctx *ns_ctx) +{ + nvme_ctrlr_process_io_completions(ns_ctx->ctr_entry->ctrlr, g_max_completions); +} + +static void +submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth) +{ + while (queue_depth-- > 0) { + submit_single_io(ns_ctx); + } +} + +static void +drain_io(struct ns_worker_ctx *ns_ctx) +{ + ns_ctx->is_draining = true; + while (ns_ctx->current_queue_depth > 0) { + check_io(ns_ctx); + } +} + +static int +work_fn(void *arg) +{ + uint64_t tsc_end = rte_get_timer_cycles() + g_time_in_sec * g_tsc_rate; + struct worker_thread *worker = (struct worker_thread *)arg; + struct ns_worker_ctx *ns_ctx = NULL; + int rc = 0; + + printf("Starting thread on core %u\n", worker->lcore); + + if (nvme_register_io_thread() != 0) { + fprintf(stderr, "nvme_register_io_thread() failed on core %u\n", worker->lcore); + return -1; + } + + /* Submit initial I/O for each namespace. */ + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + submit_io(ns_ctx, g_queue_depth); + ns_ctx = ns_ctx->next; + } + + while (1) { + /* + * Check for completed I/O for each controller. A new + * I/O will be submitted in the io_complete callback + * to replace each I/O that is completed. + */ + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + check_io(ns_ctx); + ns_ctx = ns_ctx->next; + } + + if (((tsc_end - rte_get_timer_cycles()) / g_tsc_rate) > (uint64_t)g_time_in_sec / 5 && + ((tsc_end - rte_get_timer_cycles()) / g_tsc_rate) < (uint64_t)(g_time_in_sec / 5 + 10)) { + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + rc = nvme_ctrlr_reset(ns_ctx->ctr_entry->ctrlr); + if (rc < 0) { + fprintf(stderr, "nvme reset failed.\n"); + return rc; + } + ns_ctx = ns_ctx->next; + } + } + + if (rte_get_timer_cycles() > tsc_end) { + break; + } + } + + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + drain_io(ns_ctx); + ns_ctx = ns_ctx->next; + } + + nvme_unregister_io_thread(); + + return 0; +} + +static void usage(char *program_name) +{ + printf("%s options", program_name); + printf("\n"); + printf("\t[-q io depth]\n"); + printf("\t[-s io size in bytes]\n"); + printf("\t[-w io pattern type, must be one of\n"); + printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n"); + printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n"); + printf("\t[-t time in seconds(should be larger than 15 seconds)]\n"); + printf("\t[-m max completions per poll]\n"); + printf("\t\t(default:0 - unlimited)\n"); +} + +static void +print_stats(void) +{ + uint64_t io_completed, io_submitted, io_completed_error; + uint64_t total_completed_io, total_submitted_io, total_completed_err_io; + struct worker_thread *worker; + struct ns_worker_ctx *ns_ctx; + + total_completed_io = 0; + total_submitted_io = 0; + total_completed_err_io = 0; + + worker = g_workers; + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + io_completed = ns_ctx->io_completed; + io_submitted = ns_ctx->io_submitted; + io_completed_error = ns_ctx->io_completed_error; + total_completed_io += io_completed; + total_submitted_io += io_submitted; + total_completed_err_io += io_completed_error; + ns_ctx = ns_ctx->next; + } + + if (total_submitted_io != (total_completed_io + total_completed_err_io)) { + fprintf(stderr, "Some IO are missing......\n"); + exit(1); + } + printf("========================================================\n"); + printf("%-10s: %lu completion IO %lu completion with error IO %lu submission IO\n", + "Total", total_completed_io, total_completed_err_io, total_submitted_io); +} + +static int +parse_args(int argc, char **argv) +{ + const char *workload_type; + int op; + bool mix_specified = false; + + /* default value*/ + g_queue_depth = 0; + g_io_size_bytes = 0; + workload_type = NULL; + g_time_in_sec = 0; + g_rw_percentage = -1; + + while ((op = getopt(argc, argv, "m:q:s:t:w:M:")) != -1) { + switch (op) { + case 'q': + g_queue_depth = atoi(optarg); + break; + case 's': + g_io_size_bytes = atoi(optarg); + break; + case 't': + g_time_in_sec = atoi(optarg); + break; + case 'w': + workload_type = optarg; + break; + case 'M': + g_rw_percentage = atoi(optarg); + mix_specified = true; + break; + default: + usage(argv[0]); + return 1; + } + } + + if (!g_queue_depth) { + usage(argv[0]); + return 1; + } + if (!g_io_size_bytes) { + usage(argv[0]); + return 1; + } + if (!workload_type) { + usage(argv[0]); + return 1; + } + if (!g_time_in_sec) { + usage(argv[0]); + return 1; + } + + if (strcmp(workload_type, "read") && + strcmp(workload_type, "write") && + strcmp(workload_type, "randread") && + strcmp(workload_type, "randwrite") && + strcmp(workload_type, "rw") && + strcmp(workload_type, "randrw")) { + fprintf(stderr, + "io pattern type must be one of\n" + "(read, write, randread, randwrite, rw, randrw)\n"); + return 1; + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "randread")) { + g_rw_percentage = 100; + } + + if (!strcmp(workload_type, "write") || + !strcmp(workload_type, "randwrite")) { + g_rw_percentage = 0; + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "randread") || + !strcmp(workload_type, "write") || + !strcmp(workload_type, "randwrite")) { + if (mix_specified) { + fprintf(stderr, "Ignoring -M option... Please use -M option" + " only when using rw or randrw.\n"); + } + } + + if (!strcmp(workload_type, "rw") || + !strcmp(workload_type, "randrw")) { + if (g_rw_percentage < 0 || g_rw_percentage > 100) { + fprintf(stderr, + "-M must be specified to value from 0 to 100 " + "for rw or randrw.\n"); + return 1; + } + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "write") || + !strcmp(workload_type, "rw")) { + g_is_random = 0; + } else { + g_is_random = 1; + } + + optind = 1; + return 0; +} + +static int +register_workers(void) +{ + struct worker_thread *worker; + + worker = malloc(sizeof(struct worker_thread)); + if (worker == NULL) { + perror("worker_thread malloc"); + return -1; + } + + memset(worker, 0, sizeof(struct worker_thread)); + worker->lcore = rte_get_master_lcore(); + + g_workers = worker; + g_num_workers = 1; + + return 0; +} + +static int +register_controllers(void) +{ + struct pci_device_iterator *pci_dev_iter; + struct pci_device *pci_dev; + struct pci_id_match match; + int rc; + + printf("Initializing NVMe Controllers\n"); + + pci_system_init(); + + match.vendor_id = PCI_MATCH_ANY; + match.subvendor_id = PCI_MATCH_ANY; + match.subdevice_id = PCI_MATCH_ANY; + match.device_id = PCI_MATCH_ANY; + match.device_class = NVME_CLASS_CODE; + match.device_class_mask = 0xFFFFFF; + + pci_dev_iter = pci_id_match_iterator_create(&match); + + rc = 0; + while ((pci_dev = pci_device_next(pci_dev_iter))) { + struct nvme_controller *ctrlr; + + if (pci_device_has_non_null_driver(pci_dev)) { + fprintf(stderr, "non-null kernel driver attached to nvme\n"); + fprintf(stderr, " controller at pci bdf %d:%d:%d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + fprintf(stderr, " skipping...\n"); + continue; + } + + pci_device_probe(pci_dev); + + ctrlr = nvme_attach(pci_dev); + if (ctrlr == NULL) { + fprintf(stderr, "nvme_attach failed for controller at pci bdf %d:%d:%d\n", + pci_dev->bus, pci_dev->dev, pci_dev->func); + rc = 1; + continue; + } + + register_ctrlr(ctrlr, pci_dev); + } + + pci_iterator_destroy(pci_dev_iter); + + return rc; +} + +static void +unregister_controllers(void) +{ + struct ctrlr_entry *entry = g_controllers; + + while (entry) { + struct ctrlr_entry *next = entry->next; + nvme_detach(entry->ctrlr); + free(entry); + entry = next; + } +} + +static int +associate_workers_with_ns(void) +{ + struct ns_entry *entry = g_namespaces; + struct ctrlr_entry *controller_entry = g_controllers; + struct worker_thread *worker = g_workers; + struct ns_worker_ctx *ns_ctx; + int i, count; + + count = g_num_namespaces; + + for (i = 0; i < count; i++) { + ns_ctx = malloc(sizeof(struct ns_worker_ctx)); + if (!ns_ctx) { + return -1; + } + memset(ns_ctx, 0, sizeof(*ns_ctx)); + + printf("Associating %s with lcore %d\n", entry->name, worker->lcore); + ns_ctx->entry = entry; + ns_ctx->ctr_entry = controller_entry; + ns_ctx->next = worker->ns_ctx; + worker->ns_ctx = ns_ctx; + + worker = g_workers; + + controller_entry = controller_entry->next; + entry = entry->next; + if (entry == NULL) { + entry = g_namespaces; + } + } + + return 0; +} + +static int +run_nvme_reset_cycle(int retry_count) +{ + int rc; + struct worker_thread *worker; + struct ns_worker_ctx *ns_ctx; + + nvme_retry_count = retry_count; + + rc = work_fn(g_workers); + if (rc != 0) { + return rc;; + } + + print_stats(); + + worker = g_workers; + ns_ctx = worker->ns_ctx; + while (ns_ctx != NULL) { + ns_ctx->io_completed = 0; + ns_ctx->io_completed_error = 0; + ns_ctx->io_submitted = 0; + ns_ctx->is_draining = false; + ns_ctx = ns_ctx->next; + } + + return 0; +} + +static char *ealargs[] = { + "reset", + "-c 0x1", /* This must be the second parameter. It is overwritten by index in main(). */ + "-n 4", +}; + +int main(int argc, char **argv) +{ + int rc; + int i; + + rc = parse_args(argc, argv); + if (rc != 0) { + return rc; + } + + rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs); + + if (rc < 0) { + fprintf(stderr, "could not initialize dpdk\n"); + return 1; + } + + request_mempool = rte_mempool_create("nvme_request", 8192, + nvme_request_size(), 128, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + + if (request_mempool == NULL) { + fprintf(stderr, "could not initialize request mempool\n"); + return 1; + } + + task_pool = rte_mempool_create("task_pool", 8192, + sizeof(struct reset_task), + 64, 0, NULL, NULL, task_ctor, NULL, + SOCKET_ID_ANY, 0); + + g_tsc_rate = rte_get_timer_hz(); + + if (register_workers() != 0) { + return 1; + } + + if (register_controllers() != 0) { + return 1; + } + + if (associate_workers_with_ns() != 0) { + goto error_return; + } + + printf("Initialization complete. Launching workers.\n"); + + for (i = 2; i >= 0; i--) { + rc = run_nvme_reset_cycle(i); + if (rc != 0) { + goto error_return; + } + } + + unregister_controllers(); + return 0; + +error_return: + + unregister_controllers(); + + if (rc != 0) { + fprintf(stderr, "%s: errors occured\n", argv[0]); + } + + return rc; +}