From a9be4f2c2f6e373bffbc616747f3ba670d80949b Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Thu, 20 Oct 2022 13:38:46 -0700 Subject: [PATCH] trace: add likely/unlikely hints to _spdk_trace_record This helps generate slightly better code in this function, which can have a noticeable impact for high trace event workloads. Tested with bdevperf, single malloc or null bdev, qd=32, 512B randreads on a single Xeon core. Specify "-e bdev" to enable bdev trace events. Null: Before: 8.09M/s (123ns per IO) After: 8.68M/s (115ns per IO) Malloc: Before: 4.21M/s (237ns per IO) After: 4.34M/s (230ns per IO) Note that each bdev I/O generates two trace events (START and END) - meaning this change removes 7-8ns of overhead for every 2 trace events, at least on my system. Signed-off-by: Jim Harris Change-Id: I7021b7f9e28b4a7cb16f8a97b4d4004ae165efd2 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15096 Tested-by: SPDK CI Jenkins Reviewed-by: Changpeng Liu Reviewed-by: Konrad Sztyber --- lib/trace/trace.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/trace/trace.c b/lib/trace/trace.c index 7409d1a6b..52b54de9d 100644 --- a/lib/trace/trace.c +++ b/lib/trace/trace.c @@ -12,6 +12,7 @@ #include "spdk/barrier.h" #include "spdk/log.h" #include "spdk/cpuset.h" +#include "spdk/likely.h" static int g_trace_fd = -1; static char g_shm_name[64]; @@ -39,11 +40,12 @@ _spdk_trace_record(uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, uint32_ va_list vl; lcore = spdk_env_get_current_core(); - if (lcore >= SPDK_TRACE_MAX_LCORE) { + if (spdk_unlikely(lcore >= SPDK_TRACE_MAX_LCORE)) { return; } lcore_history = spdk_get_per_lcore_history(g_trace_histories, lcore); + if (tsc == 0) { tsc = spdk_get_ticks(); } @@ -52,7 +54,7 @@ _spdk_trace_record(uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, uint32_ tpoint = &g_trace_flags->tpoint[tpoint_id]; /* Make sure that the number of arguments passed matches tracepoint definition */ - if (tpoint->num_args != num_args) { + if (spdk_unlikely(tpoint->num_args != num_args)) { assert(0 && "Unexpected number of tracepoint arguments"); return; } @@ -106,7 +108,7 @@ _spdk_trace_record(uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, uint32_ argoff = 0; while (argoff < argument->size) { /* Current buffer is full, we need to acquire another one */ - if (offset == sizeof(buffer->data)) { + if (spdk_unlikely(offset == sizeof(buffer->data))) { buffer = (struct spdk_trace_entry_buffer *) get_trace_entry( lcore_history, lcore_history->next_entry + num_entries); @@ -117,7 +119,7 @@ _spdk_trace_record(uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, uint32_ } curlen = spdk_min(sizeof(buffer->data) - offset, argument->size - argoff); - if (argoff < arglen) { + if (spdk_likely(argoff < arglen)) { assert(argval != NULL); memcpy(&buffer->data[offset], (uint8_t *)argval + argoff, spdk_min(curlen, arglen - argoff)); @@ -128,7 +130,7 @@ _spdk_trace_record(uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id, uint32_ } /* Make sure that truncated strings are NULL-terminated */ - if (argument->type == SPDK_TRACE_ARG_TYPE_STR) { + if (spdk_unlikely(argument->type == SPDK_TRACE_ARG_TYPE_STR)) { assert(offset > 0); buffer->data[offset - 1] = '\0'; }