From b482eda06ea2f6700acd2893aaa87adfb36b1345 Mon Sep 17 00:00:00 2001 From: Evgeniy Kochetov Date: Tue, 17 Nov 2020 14:20:09 +0200 Subject: [PATCH] nvme/perf: Basic synchronization of workers Currently, NVMe perf worker starts IO and measurements as soon as all its QPs were connected. But other workers may still be connecting and not started their measurements yet. With large number of QPs when connections take a long time this can cause inaccurate performance reporting. This patch adds synchronization point for workers after all QPs were connected and before start of IO and measurements. Signed-off-by: Evgeniy Kochetov Change-Id: If0c9be8dd41c8e851aae6b3e71afa3efe5314330 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5126 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris --- examples/nvme/perf/perf.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/nvme/perf/perf.c b/examples/nvme/perf/perf.c index 81b31a6f4..7a4cbaef9 100644 --- a/examples/nvme/perf/perf.c +++ b/examples/nvme/perf/perf.c @@ -222,6 +222,7 @@ static int g_num_namespaces; static TAILQ_HEAD(, worker_thread) g_workers = TAILQ_HEAD_INITIALIZER(g_workers); static int g_num_workers = 0; static uint32_t g_main_core; +static pthread_barrier_t g_worker_sync_barrier; static uint64_t g_tsc_rate; @@ -1280,15 +1281,24 @@ work_fn(void *arg) struct ns_worker_ctx *ns_ctx = NULL; uint32_t unfinished_ns_ctx; bool warmup = false; + int rc; /* Allocate queue pairs for each namespace. */ TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { if (init_ns_worker_ctx(ns_ctx) != 0) { printf("ERROR: init_ns_worker_ctx() failed\n"); + /* Wait on barrier to avoid blocking of successful workers */ + pthread_barrier_wait(&g_worker_sync_barrier); return 1; } } + rc = pthread_barrier_wait(&g_worker_sync_barrier); + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { + printf("ERROR: failed to wait on thread sync barrier\n"); + return 1; + } + tsc_current = spdk_get_ticks(); tsc_next_print = tsc_current + g_tsc_rate; @@ -2382,6 +2392,12 @@ int main(int argc, char **argv) goto cleanup; } + rc = pthread_barrier_init(&g_worker_sync_barrier, NULL, g_num_workers); + if (rc != 0) { + fprintf(stderr, "Unable to initialize thread sync barrier\n"); + goto cleanup; + } + printf("Initialization complete. Launching workers.\n"); /* Launch all of the secondary workers */ @@ -2404,6 +2420,7 @@ int main(int argc, char **argv) print_stats(); cleanup: + pthread_barrier_destroy(&g_worker_sync_barrier); if (thread_id && pthread_cancel(thread_id) == 0) { pthread_join(thread_id, NULL); }