From 038236949f6d1aa93f9d4102f96aa21ce6e20869 Mon Sep 17 00:00:00 2001 From: Cunyin Chang Date: Tue, 15 Nov 2016 13:10:32 +0800 Subject: [PATCH] example: Add example for nvme hotplug. Change-Id: I7e0bbf48798b77365c4dc968427c8834e0b05a44 Signed-off-by: Cunyin Chang --- examples/nvme/Makefile | 2 +- examples/nvme/hotplug/Makefile | 39 ++ examples/nvme/hotplug/hotplug.c | 1005 +++++++++++++++++++++++++++++++ 3 files changed, 1045 insertions(+), 1 deletion(-) create mode 100644 examples/nvme/hotplug/Makefile create mode 100644 examples/nvme/hotplug/hotplug.c diff --git a/examples/nvme/Makefile b/examples/nvme/Makefile index 254f9b17e..5adac1bb6 100644 --- a/examples/nvme/Makefile +++ b/examples/nvme/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -DIRS-y += hello_world identify perf reserve nvme_manage arbitration +DIRS-y += hello_world identify perf reserve nvme_manage arbitration hotplug DIRS-$(CONFIG_FIO_PLUGIN) += fio_plugin diff --git a/examples/nvme/hotplug/Makefile b/examples/nvme/hotplug/Makefile new file mode 100644 index 000000000..0dcdda9af --- /dev/null +++ b/examples/nvme/hotplug/Makefile @@ -0,0 +1,39 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +APP = hotplug + +include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk diff --git a/examples/nvme/hotplug/hotplug.c b/examples/nvme/hotplug/hotplug.c new file mode 100644 index 000000000..6fdf4c3f4 --- /dev/null +++ b/examples/nvme/hotplug/hotplug.c @@ -0,0 +1,1005 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "spdk/fd.h" +#include "spdk/nvme.h" +#include "spdk/env.h" +#include "spdk/string.h" +#include "spdk/nvme_intel.h" + +struct ctrlr_entry { + bool is_removed; + struct spdk_nvme_ctrlr *ctrlr; + struct spdk_nvme_intel_rw_latency_page *latency_page; + struct ctrlr_entry *next; + char name[1024]; +}; + +enum entry_type { + ENTRY_TYPE_NVME_NS, +}; + +struct ns_entry { + enum entry_type type; + + struct { + struct spdk_nvme_ctrlr *ctrlr; + struct spdk_nvme_ns *ns; + } nvme; + + struct ns_entry *next; + uint32_t io_size_blocks; + uint64_t size_in_ios; + char name[1024]; +}; + +struct ns_worker_ctx { + struct ns_entry *entry; + struct worker_thread *worker; + uint64_t io_completed; + uint64_t current_queue_depth; + uint64_t offset_in_ios; + bool is_draining; + bool is_removed; + + struct { + struct spdk_nvme_qpair *qpair; + } nvme; + + struct ns_worker_ctx *next; +}; + +struct perf_task { + struct ns_worker_ctx *ns_ctx; + void *buf; +}; + +struct worker_thread { + struct ns_worker_ctx *ns_ctx; + struct worker_thread *next; + unsigned lcore; + bool is_aborted; +}; + +static int g_outstanding_commands; + +static bool g_latency_tracking_enable = false; + +static struct rte_mempool *task_pool; + +static struct ctrlr_entry *g_controllers = NULL; +static struct ns_entry *g_namespaces = NULL; +static int g_num_namespaces = 0; +static struct worker_thread *g_workers = NULL; +static int g_num_workers = 0; + +static uint64_t g_tsc_rate; + +static uint32_t g_io_size_bytes; +static int g_rw_percentage; +static int g_is_random; +static int g_queue_depth; +static int g_time_in_sec; +static uint32_t g_max_completions; + +static const char *g_core_mask; + +static void +task_complete(struct perf_task *task); + +static void +register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) +{ + struct ns_entry *entry; + const struct spdk_nvme_ctrlr_data *cdata; + + cdata = spdk_nvme_ctrlr_get_data(ctrlr); + + if (!spdk_nvme_ns_is_active(ns)) { + printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n", + cdata->mn, cdata->sn, + spdk_nvme_ns_get_id(ns)); + return; + } + + if (spdk_nvme_ns_get_size(ns) < g_io_size_bytes || + spdk_nvme_ns_get_sector_size(ns) > g_io_size_bytes) { + printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid " + "ns size %" PRIu64 " / block size %u for I/O size %u\n", + cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns), + spdk_nvme_ns_get_size(ns), spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes); + return; + } + + entry = malloc(sizeof(struct ns_entry)); + if (entry == NULL) { + perror("ns_entry malloc"); + exit(1); + } + + entry->type = ENTRY_TYPE_NVME_NS; + entry->nvme.ctrlr = ctrlr; + entry->nvme.ns = ns; + + entry->size_in_ios = spdk_nvme_ns_get_size(ns) / + g_io_size_bytes; + entry->io_size_blocks = g_io_size_bytes / spdk_nvme_ns_get_sector_size(ns); + + snprintf(entry->name, 44, "%-20.20s (%-20.20s)", cdata->mn, cdata->sn); + + g_num_namespaces++; + entry->next = g_namespaces; + g_namespaces = entry; +} + +static void +enable_latency_tracking_complete(void *cb_arg, const struct spdk_nvme_cpl *cpl) +{ + if (spdk_nvme_cpl_is_error(cpl)) { + printf("enable_latency_tracking_complete failed\n"); + } + g_outstanding_commands--; +} + +static void +set_latency_tracking_feature(struct spdk_nvme_ctrlr *ctrlr, bool enable) +{ + int res; + union spdk_nvme_intel_feat_latency_tracking latency_tracking; + + if (enable) { + latency_tracking.bits.enable = 0x01; + } else { + latency_tracking.bits.enable = 0x00; + } + + res = spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING, + latency_tracking.raw, 0, NULL, 0, enable_latency_tracking_complete, NULL); + if (res) { + printf("fail to allocate nvme request.\n"); + return; + } + g_outstanding_commands++; + + while (g_outstanding_commands) { + spdk_nvme_ctrlr_process_admin_completions(ctrlr); + } +} + +static void +register_ctrlr(struct spdk_nvme_ctrlr *ctrlr) +{ + int nsid, num_ns; + struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry)); + const struct spdk_nvme_ctrlr_data *cdata = spdk_nvme_ctrlr_get_data(ctrlr); + + if (entry == NULL) { + perror("ctrlr_entry malloc"); + exit(1); + } + + entry->latency_page = spdk_zmalloc(sizeof(struct spdk_nvme_intel_rw_latency_page), + 4096, NULL); + if (entry->latency_page == NULL) { + printf("Allocation error (latency page)\n"); + exit(1); + } + + snprintf(entry->name, sizeof(entry->name), "%-20.20s (%-20.20s)", cdata->mn, cdata->sn); + + entry->ctrlr = ctrlr; + entry->next = g_controllers; + entry->is_removed = false; + g_controllers = entry; + + if (g_latency_tracking_enable && + spdk_nvme_ctrlr_is_feature_supported(ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) + set_latency_tracking_feature(ctrlr, true); + + num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr); + for (nsid = 1; nsid <= num_ns; nsid++) { + register_ns(ctrlr, spdk_nvme_ctrlr_get_ns(ctrlr, nsid)); + } + +} + +static void task_ctor(struct rte_mempool *mp, void *arg, void *__task, unsigned id) +{ + struct perf_task *task = __task; + task->buf = spdk_zmalloc(g_io_size_bytes, 0x200, NULL); + if (task->buf == NULL) { + fprintf(stderr, "task->buf rte_malloc failed\n"); + exit(1); + } + memset(task->buf, id % 8, g_io_size_bytes); +} + +static void io_complete(void *ctx, const struct spdk_nvme_cpl *completion); + +static __thread unsigned int seed = 0; + +static void +submit_single_io(struct ns_worker_ctx *ns_ctx) +{ + struct perf_task *task = NULL; + uint64_t offset_in_ios; + int rc; + struct ns_entry *entry = ns_ctx->entry; + + if (rte_mempool_get(task_pool, (void **)&task) != 0) { + fprintf(stderr, "task_pool rte_mempool_get failed\n"); + exit(1); + } + + task->ns_ctx = ns_ctx; + + if (g_is_random) { + offset_in_ios = rand_r(&seed) % entry->size_in_ios; + } else { + offset_in_ios = ns_ctx->offset_in_ios++; + if (ns_ctx->offset_in_ios == entry->size_in_ios) { + ns_ctx->offset_in_ios = 0; + } + } + + if ((g_rw_percentage == 100) || + (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) { + rc = spdk_nvme_ns_cmd_read(entry->nvme.ns, ns_ctx->nvme.qpair, task->buf, + offset_in_ios * entry->io_size_blocks, + entry->io_size_blocks, io_complete, task, 0); + } else { + rc = spdk_nvme_ns_cmd_write(entry->nvme.ns, ns_ctx->nvme.qpair, task->buf, + offset_in_ios * entry->io_size_blocks, + entry->io_size_blocks, io_complete, task, 0); + } + + if (rc != 0) { + fprintf(stderr, "starting I/O failed\n"); + rte_mempool_put(task_pool, task); + } else { + ns_ctx->current_queue_depth++; + } +} + +static void +task_complete(struct perf_task *task) +{ + struct ns_worker_ctx *ns_ctx; + + ns_ctx = task->ns_ctx; + ns_ctx->current_queue_depth--; + ns_ctx->io_completed++; + + rte_mempool_put(task_pool, task); + + /* + * is_draining indicates when time has expired for the test run + * and we are just waiting for the previously submitted I/O + * to complete. In this case, do not submit a new I/O to replace + * the one just completed. + */ + if (!ns_ctx->is_draining && !ns_ctx->is_removed && !ns_ctx->worker->is_aborted) { + submit_single_io(ns_ctx); + } +} + +static void +io_complete(void *ctx, const struct spdk_nvme_cpl *completion) +{ + task_complete((struct perf_task *)ctx); +} + +static void +check_io(struct ns_worker_ctx *ns_ctx) +{ + spdk_nvme_qpair_process_completions(ns_ctx->nvme.qpair, + g_max_completions); +} + +static void +submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth) +{ + while (queue_depth-- > 0) { + submit_single_io(ns_ctx); + } +} + +static void +drain_io(struct ns_worker_ctx *ns_ctx) +{ + ns_ctx->is_draining = true; + while (!ns_ctx->worker->is_aborted && ns_ctx->current_queue_depth > 0) { + check_io(ns_ctx); + } +} + +static int +init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + /* + * TODO: If a controller has multiple namespaces, they could all use the same queue. + * For now, give each namespace/thread combination its own queue. + */ + ns_ctx->nvme.qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_ctx->entry->nvme.ctrlr, 0); + if (!ns_ctx->nvme.qpair) { + printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n"); + return -1; + } + + return 0; +} + +static void +cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx) +{ + if (ns_ctx->entry->type == ENTRY_TYPE_NVME_NS) { + spdk_nvme_ctrlr_free_io_qpair(ns_ctx->nvme.qpair); + } +} + +static int +work_fn(void *arg) +{ + uint64_t tsc_end; + struct worker_thread *worker = (struct worker_thread *)arg; + struct ns_worker_ctx *ns_ctx = NULL; + + printf("Starting thread %p on core %u\n", worker, worker->lcore); + + /* Allocate a queue pair for each namespace. */ + ns_ctx = worker->ns_ctx; + while (!worker->is_aborted && ns_ctx != NULL) { + if (ns_ctx->is_removed == false) { + if (init_ns_worker_ctx(ns_ctx) != 0) { + printf("ERROR: init_ns_worker_ctx() failed\n"); + return 1; + } + } + ns_ctx = ns_ctx->next; + } + + tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate; + /* Submit initial I/O for each namespace. */ + ns_ctx = worker->ns_ctx; + + while (!worker->is_aborted && ns_ctx != NULL) { + if (ns_ctx->is_removed == false) { + submit_io(ns_ctx, g_queue_depth); + } + ns_ctx = ns_ctx->next; + } + + while (!worker->is_aborted) { + /* + * Check for completed I/O for each controller. A new + * I/O will be submitted in the io_complete callback + * to replace each I/O that is completed. + */ + ns_ctx = worker->ns_ctx; + while (!worker->is_aborted && ns_ctx != NULL) { + if (ns_ctx->is_removed == false) { + check_io(ns_ctx); + } + ns_ctx = ns_ctx->next; + } + if (spdk_get_ticks() > tsc_end) { + break; + } + } + + ns_ctx = worker->ns_ctx; + while (!worker->is_aborted && ns_ctx != NULL) { + if (ns_ctx->is_removed == false) { + drain_io(ns_ctx); + } + cleanup_ns_worker_ctx(ns_ctx); + ns_ctx = ns_ctx->next; + } + printf("finish thread %p on core %u\n", worker, worker->lcore); + return 0; +} + +static void usage(char *program_name) +{ + printf("%s options", program_name); + printf("\n"); + printf("\t[-q io depth]\n"); + printf("\t[-s io size in bytes]\n"); + printf("\t[-w io pattern type, must be one of\n"); + printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n"); + printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n"); + printf("\t[-l enable latency tracking, default: disabled]\n"); + printf("\t[-t time in seconds]\n"); + printf("\t[-c core mask for I/O submission/completion.]\n"); + printf("\t\t(default: 1)]\n"); + printf("\t[-m max completions per poll]\n"); + printf("\t\t(default: 0 - unlimited)\n"); +} + +static void +print_performance(void) +{ + float io_per_second, mb_per_second; + float total_io_per_second, total_mb_per_second; + struct worker_thread *worker; + struct ns_worker_ctx *ns_ctx; + + total_io_per_second = 0; + total_mb_per_second = 0; + + worker = g_workers; + while (worker) { + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + io_per_second = (float)ns_ctx->io_completed / g_time_in_sec; + mb_per_second = io_per_second * g_io_size_bytes / (1024 * 1024); + printf("%-43.43s from core %u: %10.2f IO/s %10.2f MB/s\n", + ns_ctx->entry->name, worker->lcore, + io_per_second, mb_per_second); + total_io_per_second += io_per_second; + total_mb_per_second += mb_per_second; + ns_ctx = ns_ctx->next; + } + worker = worker->next; + } + printf("========================================================\n"); + printf("%-55s: %10.2f IO/s %10.2f MB/s\n", + "Total", total_io_per_second, total_mb_per_second); + + printf("\n"); +} + +static void +print_latency_page(struct ctrlr_entry *entry) +{ + int i; + + printf("\n"); + printf("%s\n", entry->name); + printf("--------------------------------------------------------\n"); + + for (i = 0; i < 32; i++) { + if (entry->latency_page->buckets_32us[i]) + printf("Bucket %dus - %dus: %d\n", i * 32, (i + 1) * 32, entry->latency_page->buckets_32us[i]); + } + for (i = 0; i < 31; i++) { + if (entry->latency_page->buckets_1ms[i]) + printf("Bucket %dms - %dms: %d\n", i + 1, i + 2, entry->latency_page->buckets_1ms[i]); + } + for (i = 0; i < 31; i++) { + if (entry->latency_page->buckets_32ms[i]) + printf("Bucket %dms - %dms: %d\n", (i + 1) * 32, (i + 2) * 32, + entry->latency_page->buckets_32ms[i]); + } +} + +static void +print_latency_statistics(const char *op_name, enum spdk_nvme_intel_log_page log_page) +{ + struct ctrlr_entry *ctrlr; + + printf("%s Latency Statistics:\n", op_name); + printf("========================================================\n"); + g_outstanding_commands = 0; + ctrlr = g_controllers; + while (ctrlr && !ctrlr->is_removed) { + if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) { + if (spdk_nvme_ctrlr_cmd_get_log_page(ctrlr->ctrlr, log_page, SPDK_NVME_GLOBAL_NS_TAG, + ctrlr->latency_page, sizeof(struct spdk_nvme_intel_rw_latency_page), + enable_latency_tracking_complete, + NULL)) { + printf("nvme_ctrlr_cmd_get_log_page() failed\n"); + exit(1); + } + + g_outstanding_commands++; + } else { + printf("Controller %s: %s latency statistics not supported\n", ctrlr->name, op_name); + } + ctrlr = ctrlr->next; + } + + while (g_outstanding_commands) { + ctrlr = g_controllers; + while (ctrlr) { + spdk_nvme_ctrlr_process_admin_completions(ctrlr->ctrlr); + ctrlr = ctrlr->next; + } + } + + ctrlr = g_controllers; + while (ctrlr && !ctrlr->is_removed) { + if (spdk_nvme_ctrlr_is_log_page_supported(ctrlr->ctrlr, log_page)) { + print_latency_page(ctrlr); + } + ctrlr = ctrlr->next; + } + printf("\n"); +} + +static void +print_stats(void) +{ + print_performance(); + if (g_latency_tracking_enable) { + if (g_rw_percentage != 0) { + print_latency_statistics("Read", SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY); + } + if (g_rw_percentage != 100) { + print_latency_statistics("Write", SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY); + } + } +} + +static int +parse_args(int argc, char **argv) +{ + const char *workload_type; + int op; + bool mix_specified = false; + + /* default value*/ + g_queue_depth = 0; + g_io_size_bytes = 0; + workload_type = NULL; + g_time_in_sec = 0; + g_rw_percentage = -1; + g_core_mask = NULL; + g_max_completions = 0; + + while ((op = getopt(argc, argv, "c:lm:q:s:t:w:M:")) != -1) { + switch (op) { + case 'c': + g_core_mask = optarg; + break; + case 'l': + g_latency_tracking_enable = true; + break; + case 'm': + g_max_completions = atoi(optarg); + break; + case 'q': + g_queue_depth = atoi(optarg); + break; + case 's': + g_io_size_bytes = atoi(optarg); + break; + case 't': + g_time_in_sec = atoi(optarg); + break; + case 'w': + workload_type = optarg; + break; + case 'M': + g_rw_percentage = atoi(optarg); + mix_specified = true; + break; + default: + usage(argv[0]); + return 1; + } + } + + if (!g_queue_depth) { + usage(argv[0]); + return 1; + } + if (!g_io_size_bytes) { + usage(argv[0]); + return 1; + } + if (!workload_type) { + usage(argv[0]); + return 1; + } + if (!g_time_in_sec) { + usage(argv[0]); + return 1; + } + + if (strcmp(workload_type, "read") && + strcmp(workload_type, "write") && + strcmp(workload_type, "randread") && + strcmp(workload_type, "randwrite") && + strcmp(workload_type, "rw") && + strcmp(workload_type, "randrw")) { + fprintf(stderr, + "io pattern type must be one of\n" + "(read, write, randread, randwrite, rw, randrw)\n"); + return 1; + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "randread")) { + g_rw_percentage = 100; + } + + if (!strcmp(workload_type, "write") || + !strcmp(workload_type, "randwrite")) { + g_rw_percentage = 0; + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "randread") || + !strcmp(workload_type, "write") || + !strcmp(workload_type, "randwrite")) { + if (mix_specified) { + fprintf(stderr, "Ignoring -M option... Please use -M option" + " only when using rw or randrw.\n"); + } + } + + if (!strcmp(workload_type, "rw") || + !strcmp(workload_type, "randrw")) { + if (g_rw_percentage < 0 || g_rw_percentage > 100) { + fprintf(stderr, + "-M must be specified to value from 0 to 100 " + "for rw or randrw.\n"); + return 1; + } + } + + if (!strcmp(workload_type, "read") || + !strcmp(workload_type, "write") || + !strcmp(workload_type, "rw")) { + g_is_random = 0; + } else { + g_is_random = 1; + } + + optind = 1; + return 0; +} + +static int +register_workers(void) +{ + unsigned lcore; + struct worker_thread *worker = NULL; + + g_num_workers = 0; + + RTE_LCORE_FOREACH_SLAVE(lcore) { + worker = malloc(sizeof(struct worker_thread)); + if (worker == NULL) { + perror("worker_thread malloc"); + return -1; + } + memset(worker, 0, sizeof(struct worker_thread)); + worker->lcore = lcore; + worker->is_aborted = false; + if (g_workers == NULL) { + g_workers = worker; + } else { + worker->next = g_workers; + g_workers = worker; + } + g_num_workers++; + } + + return 0; +} + +static bool +probe_cb(void *cb_ctx, const struct spdk_nvme_probe_info *probe_info, + struct spdk_nvme_ctrlr_opts *opts) +{ + printf("Attaching to %04x:%02x:%02x.%02x\n", + probe_info->pci_addr.domain, + probe_info->pci_addr.bus, + probe_info->pci_addr.dev, + probe_info->pci_addr.func); + + return true; +} + +static void +attach_cb(void *cb_ctx, const struct spdk_nvme_probe_info *probe_info, + struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) +{ + printf("Attached to %04x:%02x:%02x.%02x\n", + probe_info->pci_addr.domain, + probe_info->pci_addr.bus, + probe_info->pci_addr.dev, + probe_info->pci_addr.func); + + register_ctrlr(ctrlr); +} + +static void +remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) +{ + struct worker_thread *worker = g_workers; + struct ctrlr_entry *entry = g_controllers; + bool all_abort = true; + struct spdk_nvme_ctrlr *target_ctrlr; + struct ns_worker_ctx *ns_ctx; + + while (entry) { + if (entry->ctrlr == ctrlr) { + entry->is_removed = true; + break; + } + entry = entry->next; + } + + while (worker != NULL) { + if (!worker->ns_ctx) { + worker = worker->next; + continue; + } + + all_abort = true; + ns_ctx = worker->ns_ctx; + while (ns_ctx) { + target_ctrlr = worker->ns_ctx->entry->nvme.ctrlr; + if (ctrlr == target_ctrlr) { + ns_ctx->is_removed = true; + } else if (!ns_ctx->is_removed) { + all_abort = false; + } + ns_ctx = ns_ctx->next; + } + if (all_abort == true) { + worker->is_aborted = true; + } + worker = worker->next; + } + sleep(2); +} + +static int +register_controllers(void) +{ + printf("Initializing NVMe Controllers\n"); + + if (spdk_nvme_probe(NULL, probe_cb, attach_cb, remove_cb) != 0) { + fprintf(stderr, "spdk_nvme_probe() failed\n"); + return 1; + } + + return 0; +} + +static void +unregister_controllers(void) +{ + struct ctrlr_entry *entry = g_controllers; + + while (entry) { + struct ctrlr_entry *next = entry->next; + spdk_free(entry->latency_page); + if (!entry->is_removed) { + if (g_latency_tracking_enable && + spdk_nvme_ctrlr_is_feature_supported(entry->ctrlr, SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING)) + set_latency_tracking_feature(entry->ctrlr, false); + spdk_nvme_detach(entry->ctrlr); + } + free(entry); + entry = next; + } +} + +static void +unregister_namespaces(void) +{ + struct ns_entry *entry = g_namespaces; + + while (entry) { + struct ns_entry *next = entry->next; + free(entry); + entry = next; + } +} + +static void +unregister_workers(void) +{ + struct worker_thread *worker = g_workers; + + /* Free namespace context and worker thread */ + while (worker) { + struct worker_thread *next_worker = worker->next; + struct ns_worker_ctx *ns_ctx = worker->ns_ctx; + + while (ns_ctx) { + struct ns_worker_ctx *next_ns_ctx = ns_ctx->next; + free(ns_ctx); + ns_ctx = next_ns_ctx; + } + free(worker); + worker = next_worker; + } +} + +static int +associate_workers_with_ns(void) +{ + struct ns_entry *entry = g_namespaces; + struct worker_thread *worker = g_workers; + struct ns_worker_ctx *ns_ctx; + int i, count; + + count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers; + + for (i = 0; i < count; i++) { + ns_ctx = malloc(sizeof(struct ns_worker_ctx)); + if (!ns_ctx) { + return -1; + } + memset(ns_ctx, 0, sizeof(*ns_ctx)); + + printf("Associating %s with lcore %d\n", entry->name, worker->lcore); + ns_ctx->is_removed = false; + ns_ctx->entry = entry; + ns_ctx->next = worker->ns_ctx; + ns_ctx->worker = worker; + worker->ns_ctx = ns_ctx; + + worker = worker->next; + if (worker == NULL) { + worker = g_workers; + } + + entry = entry->next; + if (entry == NULL) { + entry = g_namespaces; + } + } + + return 0; +} + +static char *ealargs[] = { + "perf", + "-c 0x3", + "-n 4", +}; + +static void detect_hotplug(void) +{ + uint64_t tsc_end; + + tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate; + + while (1) { + if ((rte_eal_process_type() == RTE_PROC_PRIMARY)) { + if (spdk_nvme_probe(NULL, probe_cb, attach_cb, remove_cb) != 0) { + fprintf(stderr, "spdk_nvme_probe() failed\n"); + return; + } + usleep(1000); + } + if (spdk_get_ticks() > tsc_end) { + break; + } + } +} + +int main(int argc, char **argv) +{ + int rc; + struct worker_thread *worker; + struct rlimit core_limits; + + core_limits.rlim_cur = core_limits.rlim_max = RLIM_INFINITY; + setrlimit(RLIMIT_CORE, &core_limits); + + rc = parse_args(argc, argv); + if (rc != 0) { + return rc; + } + + ealargs[1] = spdk_sprintf_alloc("-c %s", g_core_mask ? g_core_mask : "0x3"); + if (ealargs[1] == NULL) { + perror("ealargs spdk_sprintf_alloc"); + return 1; + } + + rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs); + + free(ealargs[1]); + + if (rc < 0) { + fprintf(stderr, "could not initialize dpdk\n"); + return 1; + } + + task_pool = rte_mempool_create("task_pool", 8192, + sizeof(struct perf_task), + 64, 0, NULL, NULL, task_ctor, NULL, + SOCKET_ID_ANY, 0); + + g_tsc_rate = spdk_get_ticks_hz(); + + if (register_workers() != 0) { + return 1; + } + + if (register_controllers() != 0) { + return 1; + } + + if (associate_workers_with_ns() != 0) { + return 1; + } + + printf("Initialization complete. Launching workers.\n"); + + /* Launch all of the slave workers */ + worker = g_workers; + while (worker != NULL) { + rte_eal_remote_launch(work_fn, worker, worker->lcore); + worker = worker->next; + } + + detect_hotplug(); + + worker = g_workers; + while (worker != NULL) { + if (rte_eal_wait_lcore(worker->lcore) < 0) { + rc = -1; + } + printf("worker %p finished\n", worker); + worker = worker->next; + } + + print_stats(); + unregister_namespaces(); + unregister_controllers(); + unregister_workers(); + + return rc; +}