diff --git a/CONFIG b/CONFIG index 2142d9c12..c41b6890e 100644 --- a/CONFIG +++ b/CONFIG @@ -37,6 +37,9 @@ CONFIG_PREFIX?=/usr/local # Build with debug logging. Turn off for performance testing and normal usage CONFIG_DEBUG?=n +# Show backtrace when logging message at level <= lvl (ERROR, WARN, NOTICE, DEBUG) +#CONFIG_LOG_BACKTRACE?=lvl + # Treat warnings as errors (fail the build on any warning). CONFIG_WERROR?=n diff --git a/configure b/configure index b50cf77cd..81c651926 100755 --- a/configure +++ b/configure @@ -16,6 +16,8 @@ function usage() echo " --prefix=path Configure installation prefix (default: /usr/local)" echo "" echo " --enable-debug Configure for debug builds" + echo " --enable-log-bt=lvl Show backtrace using libunwind when logging message at level <= lvl." + echo " Valid values are: ERROR, WARN, NOTICE, DEBUG." echo " --enable-werror Treat compiler warnings as errors" echo " --enable-asan Enable address sanitizer" echo " --enable-ubsan Enable undefined behavior sanitizer" @@ -89,6 +91,9 @@ for i in "$@"; do --disable-debug) CONFIG_DEBUG=n ;; + --enable-log-bt=*) + CONFIG_LOG_BACKTRACE=${i#*=} + ;; --enable-asan) CONFIG_ASAN=y ;; @@ -295,6 +300,9 @@ fi if [ -n "$CONFIG_DEBUG" ]; then echo "CONFIG_DEBUG?=$CONFIG_DEBUG" >> CONFIG.local fi +if [ -n "$CONFIG_LOG_BACKTRACE" ]; then + echo "CONFIG_LOG_BACKTRACE?=$CONFIG_LOG_BACKTRACE" >> CONFIG.local +fi if [ -n "$CONFIG_WERROR" ]; then echo "CONFIG_WERROR?=$CONFIG_WERROR" >> CONFIG.local fi diff --git a/lib/log/log.c b/lib/log/log.c index 19b205dfd..fb270e2c9 100644 --- a/lib/log/log.c +++ b/lib/log/log.c @@ -35,6 +35,11 @@ #include "spdk_internal/log.h" +#ifdef SPDK_LOG_BACKTRACE_LVL +#define UNW_LOCAL_ONLY +#include +#endif + static const char *const spdk_level_names[] = { [SPDK_LOG_ERROR] = "ERROR", [SPDK_LOG_WARN] = "WARNING", @@ -57,6 +62,44 @@ spdk_log_close(void) closelog(); } +#ifdef SPDK_LOG_BACKTRACE_LVL +static void +spdk_log_unwind_stack(FILE *fp, enum spdk_log_level level) +{ + unw_error_t err; + unw_cursor_t cursor; + unw_context_t uc; + unw_word_t ip; + unw_word_t offp; + char f_name[64]; + int frame; + + if (level > SPDK_LOG_BACKTRACE_LVL) { + return; + } + + unw_getcontext(&uc); + unw_init_local(&cursor, &uc); + fprintf(fp, "*%s*: === BACKTRACE START ===\n", spdk_level_names[level]); + + unw_step(&cursor); + for (frame = 1; unw_step(&cursor) > 0; frame++) { + unw_get_reg(&cursor, UNW_REG_IP, &ip); + err = unw_get_proc_name(&cursor, f_name, sizeof(f_name), &offp); + if (err || strcmp(f_name, "main") == 0) { + break; + } + + fprintf(fp, "*%s*: %3d: %*s%s() at %#lx\n", spdk_level_names[level], frame, frame - 1, "", f_name, + (unsigned long)ip); + } + fprintf(fp, "*%s*: === BACKTRACE END ===\n", spdk_level_names[level]); +} + +#else +#define spdk_log_unwind_stack(fp, lvl) +#endif + void spdk_log(enum spdk_log_level level, const char *file, const int line, const char *func, const char *format, ...) @@ -87,6 +130,7 @@ spdk_log(enum spdk_log_level level, const char *file, const int line, const char if (level <= g_spdk_log_print_level) { fprintf(stderr, "%s:%4d:%s: *%s*: %s", file, line, func, spdk_level_names[level], buf); + spdk_log_unwind_stack(stderr, level); } if (level <= g_spdk_log_level) { diff --git a/mk/spdk.common.mk b/mk/spdk.common.mk index 98a8c32ab..461961b0b 100644 --- a/mk/spdk.common.mk +++ b/mk/spdk.common.mk @@ -193,6 +193,10 @@ CXXFLAGS += $(COMMON_CFLAGS) -std=c++0x SYS_LIBS += -lrt SYS_LIBS += -luuid SYS_LIBS += -lcrypto +ifneq ($(CONFIG_LOG_BACKTRACE),) +SYS_LIBS += -lunwind +COMMON_CFLAGS += -DSPDK_LOG_BACKTRACE_LVL=SPDK_LOG_$(CONFIG_LOG_BACKTRACE) +endif MAKEFLAGS += --no-print-directory diff --git a/scripts/pkgdep.sh b/scripts/pkgdep.sh index da5f77d77..52d054d00 100755 --- a/scripts/pkgdep.sh +++ b/scripts/pkgdep.sh @@ -13,6 +13,8 @@ if [ -s /etc/redhat-release ]; then git astyle python-pep8 lcov python clang-analyzer libuuid-devel \ sg3_utils libiscsi-devel yum install -y --allowerasing openssl-devel + # Additional (optional) dependencies for showing backtrace in logs + yum install libunwind-devel # Additional dependencies for NVMe over Fabrics yum install -y libibverbs-devel librdmacm-devel # Additional dependencies for DPDK @@ -27,6 +29,8 @@ elif [ -f /etc/debian_version ]; then # Includes Ubuntu, Debian apt-get install -y gcc g++ make libcunit1-dev libaio-dev libssl-dev \ git astyle pep8 lcov clang uuid-dev sg3-utils libiscsi-dev + # Additional (optional) dependencies for showing backtrace in logs + apt-get install libunwind-dev # Additional dependencies for NVMe over Fabrics apt-get install -y libibverbs-dev librdmacm-dev # Additional dependencies for DPDK @@ -40,6 +44,8 @@ elif [ -f /etc/debian_version ]; then elif [ -f /etc/SuSE-release ]; then zypper install -y gcc gcc-c++ make cunit-devel libaio-devel libopenssl-devel \ git-core lcov python-base python-pep8 libuuid-devel sg3_utils + # Additional (optional) dependencies for showing backtrace in logs + zypper install libunwind-devel # Additional dependencies for NVMe over Fabrics zypper install -y rdma-core-devel # Additional dependencies for DPDK diff --git a/test/common/autotest_common.sh b/test/common/autotest_common.sh index 00a8b4203..ed4b9eb11 100755 --- a/test/common/autotest_common.sh +++ b/test/common/autotest_common.sh @@ -74,6 +74,11 @@ fi config_params='--enable-debug --enable-werror' +if echo -e "#include \nint main(int argc, char *argv[]) {return 0;}\n" | \ + gcc -o /dev/null -lunwind -x c - 2>/dev/null; then + config_params+=' --enable-log-bt=ERROR' +fi + export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1' # On Linux systems, override the default HUGEMEM in scripts/setup.sh to diff --git a/test/nvme/spdk_nvme_cli.sh b/test/nvme/spdk_nvme_cli.sh index 9984f8c4a..001f26b68 100755 --- a/test/nvme/spdk_nvme_cli.sh +++ b/test/nvme/spdk_nvme_cli.sh @@ -32,8 +32,13 @@ ln -sf "$rootdir" "$spdk_nvme_cli/spdk" bdfs=$(iter_pci_class_code 01 08 02) bdf=$(echo $bdfs|awk '{ print $1 }') + +if [[ -s $rootdir/CONFIG.local ]] && grep CONFIG_LOG_BACKTRACE $rootdir/CONFIG.local -q; then + nvme_cli_ldflags='LDFLAGS=-lunwind' +fi + cd $spdk_nvme_cli -make clean && make +make clean && make -j$(nproc) $nvme_cli_ldflags sed -i 's/spdk=0/spdk=1/g' spdk.conf sed -i 's/shm_id=1/shm_id=0/g' spdk.conf ./nvme list